Retry certificate renewal after failure

database/certificate.sql.go

@@ -75,7 +75,7 @@ FROM certificates AS cert
 WHERE cert.active = 1
   AND (cert.auto_renew = 1 OR cert.not_after IS NULL)
   AND cert.renewing = 0
-  AND cert.renew_failed = 0
+  AND DATETIME() > DATETIME(cert.renew_retry)
   AND (cert.not_after IS NULL OR DATETIME(cert.not_after, 'utc', '-30 days') < DATETIME())
 ORDER BY cert.temp_parent, cert.not_after DESC NULLS FIRST
 LIMIT 1
@@ -107,7 +107,7 @@ SELECT cert.id,
        cert.auto_renew,
        cert.active,
        cert.renewing,
-       cert.renew_failed,
+       cert.renew_retry,
        cert.not_after,
        cert.updated_at,
        certificate_domains.domain
@@ -120,7 +120,7 @@ type FindOwnedCertsRow struct {
 	AutoRenew  bool      `json:"auto_renew"`
 	Active     bool      `json:"active"`
 	Renewing   bool      `json:"renewing"`
-	RenewFailed bool      `json:"renew_failed"`
+	RenewRetry time.Time `json:"renew_retry"`
 	NotAfter   time.Time `json:"not_after"`
 	UpdatedAt  time.Time `json:"updated_at"`
 	Domain     string    `json:"domain"`
@@ -140,7 +140,7 @@ func (q *Queries) FindOwnedCerts(ctx context.Context) ([]FindOwnedCertsRow, erro
 			&i.AutoRenew,
 			&i.Active,
 			&i.Renewing,
-			&i.RenewFailed,
+			&i.RenewRetry,
 			&i.NotAfter,
 			&i.UpdatedAt,
 			&i.Domain,
@@ -169,10 +169,21 @@ func (q *Queries) RemoveCertificate(ctx context.Context, id int64) error {
 	return err
 }
 
+const setRetryFlag = `-- name: SetRetryFlag :exec
+UPDATE certificates
+SET renew_retry = DATETIME('now', '+1 day')
+WHERE id = ?
+`
+
+func (q *Queries) SetRetryFlag(ctx context.Context, id int64) error {
+	_, err := q.db.ExecContext(ctx, setRetryFlag, id)
+	return err
+}
+
 const updateCertAfterRenewal = `-- name: UpdateCertAfterRenewal :exec
 UPDATE certificates
 SET renewing   = 0,
-    renew_failed=0,
+    renew_retry=0,
     not_after=?,
     updated_at=?
 WHERE id = ?
@@ -192,17 +203,17 @@ func (q *Queries) UpdateCertAfterRenewal(ctx context.Context, arg UpdateCertAfte
 const updateRenewingState = `-- name: UpdateRenewingState :exec
 UPDATE certificates
 SET renewing    = ?,
-    renew_failed = ?
+    renew_retry = ?
 WHERE id = ?
 `
 
 type UpdateRenewingStateParams struct {
 	Renewing   bool      `json:"renewing"`
-	RenewFailed bool  `json:"renew_failed"`
+	RenewRetry time.Time `json:"renew_retry"`
 	ID         int64     `json:"id"`
 }
 
 func (q *Queries) UpdateRenewingState(ctx context.Context, arg UpdateRenewingStateParams) error {
-	_, err := q.db.ExecContext(ctx, updateRenewingState, arg.Renewing, arg.RenewFailed, arg.ID)
+	_, err := q.db.ExecContext(ctx, updateRenewingState, arg.Renewing, arg.RenewRetry, arg.ID)
 	return err
 }

database/migrations/20240920175046_retry_renewal.up.sql

@@ -0,0 +1,5 @@
+ALTER TABLE certificates
+    DROP COLUMN renew_failed;
+
+ALTER TABLE certificates
+    ADD COLUMN renew_retry DATETIME NOT NULL DEFAULT 0;

database/models.go

@@ -16,10 +16,10 @@ type Certificate struct {
 	AutoRenew  bool          `json:"auto_renew"`
 	Active     bool          `json:"active"`
 	Renewing   bool          `json:"renewing"`
-	RenewFailed bool          `json:"renew_failed"`
 	NotAfter   time.Time     `json:"not_after"`
 	UpdatedAt  time.Time     `json:"updated_at"`
 	TempParent sql.NullInt64 `json:"temp_parent"`
+	RenewRetry time.Time     `json:"renew_retry"`
 }
 
 type CertificateDomain struct {

database/queries/certificate.sql

@@ -5,7 +5,7 @@ FROM certificates AS cert
 WHERE cert.active = 1
   AND (cert.auto_renew = 1 OR cert.not_after IS NULL)
   AND cert.renewing = 0
-  AND cert.renew_failed = 0
+  AND DATETIME() > DATETIME(cert.renew_retry)
   AND (cert.not_after IS NULL OR DATETIME(cert.not_after, 'utc', '-30 days') < DATETIME())
 ORDER BY cert.temp_parent, cert.not_after DESC NULLS FIRST
 LIMIT 1;
@@ -15,7 +15,7 @@ SELECT cert.id,
        cert.auto_renew,
        cert.active,
        cert.renewing,
-       cert.renew_failed,
+       cert.renew_retry,
        cert.not_after,
        cert.updated_at,
        certificate_domains.domain
@@ -25,13 +25,18 @@ FROM certificates AS cert
 -- name: UpdateRenewingState :exec
 UPDATE certificates
 SET renewing    = ?,
-    renew_failed = ?
+    renew_retry = ?
+WHERE id = ?;
+
+-- name: SetRetryFlag :exec
+UPDATE certificates
+SET renew_retry = DATETIME('now', '+1 day')
 WHERE id = ?;
 
 -- name: UpdateCertAfterRenewal :exec
 UPDATE certificates
 SET renewing   = 0,
-    renew_failed=0,
+    renew_retry=0,
     not_after=?,
     updated_at=?
 WHERE id = ?;

renewal/service.go

@@ -399,13 +399,14 @@ func (s *Service) getPrivateKey(id int64) (*rsa.PrivateKey, error) {
 // certificate to the certDir directory.
 func (s *Service) renewCert(localData *localCertData) error {
 	// database synchronous state
-	s.setRenewing(localData.id, true, false)
+	s.setRenewing(localData.id, true)
 	Logger.Debug("No certificates to update")
 
 	// run internal renewal code and log errors
 	cert, certBytes, err := s.renewCertInternal(localData)
 	if err != nil {
-		s.setRenewing(localData.id, false, true)
+		s.setRenewing(localData.id, false)
+		s.setRetry(localData.id)
 		return fmt.Errorf("failed to renew cert %d: %w", localData.id, err)
 	}
 
@@ -501,14 +502,20 @@ func (s *Service) renewCertInternal(localData *localCertData) (*x509.Certificate
 
 // setRenewing sets the renewing and failed states in the database for a
 // specified certificate id.
-func (s *Service) setRenewing(id int64, renewing, failed bool) {
+func (s *Service) setRenewing(id int64, renewing bool) {
 	err := s.db.UpdateRenewingState(context.Background(), database.UpdateRenewingStateParams{
 		Renewing: renewing,
-		RenewFailed: failed,
 		ID:       id,
 	})
 	if err != nil {
-		Logger.Warn("Failed to set renewing/failed mode in database", "id", id, "err", err)
+		Logger.Warn("Failed to set renewing mode in database", "id", id, "err", err)
+	}
+}
+
+func (s *Service) setRetry(id int64) {
+	err := s.db.SetRetryFlag(context.Background(), id)
+	if err != nil {
+		Logger.Warn("Failed to set retry time in database", "id", id, "err", err)
 	}
 }
 

renewal/service_test.go

@@ -170,7 +170,7 @@ func TestPebbleRenewal(t *testing.T) {
 			_, err := db2.Exec("DELETE FROM certificate_domains")
 			assert.NoError(t, err)
 
-			_, err = db2.Exec(`INSERT INTO certificates (owner, dns, auto_renew, active, renewing, renew_failed, not_after, updated_at) VALUES (1, 1, 1, 1, 0, 0, "2000-01-01 00:00:00+00:00", "2000-01-01 00:00:00+00:00")`)
+			_, err = db2.Exec(`INSERT INTO certificates (owner, dns, auto_renew, active, renewing, renew_retry, not_after, updated_at) VALUES (1, 1, 1, 1, 0, '2000-01-01 00:00:00+00:00', '2000-01-01 00:00:00+00:00', '2000-01-01 00:00:00+00:00');`)
 			assert.NoError(t, err)
 			for _, j := range i.domains {
 				_, err = db2.Exec(`INSERT INTO certificate_domains (cert_id, domain) VALUES (1, ?)`, j)

servers/api.go

@@ -29,7 +29,7 @@ type Certificate struct {
 	AutoRenew  bool      `json:"auto_renew"`
 	Active     bool      `json:"active"`
 	Renewing   bool      `json:"renewing"`
-	RenewFailed bool      `json:"renew_failed"`
+	RenewRetry time.Time `json:"renew_retry"`
 	NotAfter   time.Time `json:"not_after"`
 	UpdatedAt  time.Time `json:"updated_at"`
 	Domains    []string  `json:"domains"`
@@ -73,7 +73,7 @@ func NewApiServer(listen string, db *database.Queries, signer *mjwt.KeyStore, do
 				AutoRenew:  row.AutoRenew,
 				Active:     row.Active,
 				Renewing:   row.Renewing,
-				RenewFailed: row.RenewFailed,
+				RenewRetry: row.RenewRetry,
 				NotAfter:   row.NotAfter,
 				UpdatedAt:  row.UpdatedAt,
 			}