From d0fc76cd738d2d130f54e7e75b674f7763c56414 Mon Sep 17 00:00:00 2001 From: MrMelon54 Date: Sat, 21 Sep 2024 13:11:51 +0100 Subject: [PATCH] Retry certificate renewal after failure --- database/certificate.sql.go | 49 ++++++++++++------- .../20240920175046_retry_renewal.down.sql | 0 .../20240920175046_retry_renewal.up.sql | 5 ++ database/models.go | 20 ++++---- database/queries/certificate.sql | 17 ++++--- renewal/service.go | 21 +++++--- renewal/service_test.go | 2 +- servers/api.go | 30 ++++++------ 8 files changed, 86 insertions(+), 58 deletions(-) create mode 100644 database/migrations/20240920175046_retry_renewal.down.sql create mode 100644 database/migrations/20240920175046_retry_renewal.up.sql diff --git a/database/certificate.sql.go b/database/certificate.sql.go index de6be34..2ec4e30 100644 --- a/database/certificate.sql.go +++ b/database/certificate.sql.go @@ -75,7 +75,7 @@ FROM certificates AS cert WHERE cert.active = 1 AND (cert.auto_renew = 1 OR cert.not_after IS NULL) AND cert.renewing = 0 - AND cert.renew_failed = 0 + AND DATETIME() > DATETIME(cert.renew_retry) AND (cert.not_after IS NULL OR DATETIME(cert.not_after, 'utc', '-30 days') < DATETIME()) ORDER BY cert.temp_parent, cert.not_after DESC NULLS FIRST LIMIT 1 @@ -107,7 +107,7 @@ SELECT cert.id, cert.auto_renew, cert.active, cert.renewing, - cert.renew_failed, + cert.renew_retry, cert.not_after, cert.updated_at, certificate_domains.domain @@ -116,14 +116,14 @@ FROM certificates AS cert ` type FindOwnedCertsRow struct { - ID int64 `json:"id"` - AutoRenew bool `json:"auto_renew"` - Active bool `json:"active"` - Renewing bool `json:"renewing"` - RenewFailed bool `json:"renew_failed"` - NotAfter time.Time `json:"not_after"` - UpdatedAt time.Time `json:"updated_at"` - Domain string `json:"domain"` + ID int64 `json:"id"` + AutoRenew bool `json:"auto_renew"` + Active bool `json:"active"` + Renewing bool `json:"renewing"` + RenewRetry time.Time `json:"renew_retry"` + NotAfter time.Time `json:"not_after"` + UpdatedAt time.Time `json:"updated_at"` + Domain string `json:"domain"` } func (q *Queries) FindOwnedCerts(ctx context.Context) ([]FindOwnedCertsRow, error) { @@ -140,7 +140,7 @@ func (q *Queries) FindOwnedCerts(ctx context.Context) ([]FindOwnedCertsRow, erro &i.AutoRenew, &i.Active, &i.Renewing, - &i.RenewFailed, + &i.RenewRetry, &i.NotAfter, &i.UpdatedAt, &i.Domain, @@ -169,10 +169,21 @@ func (q *Queries) RemoveCertificate(ctx context.Context, id int64) error { return err } +const setRetryFlag = `-- name: SetRetryFlag :exec +UPDATE certificates +SET renew_retry = DATETIME('now', '+1 day') +WHERE id = ? +` + +func (q *Queries) SetRetryFlag(ctx context.Context, id int64) error { + _, err := q.db.ExecContext(ctx, setRetryFlag, id) + return err +} + const updateCertAfterRenewal = `-- name: UpdateCertAfterRenewal :exec UPDATE certificates -SET renewing = 0, - renew_failed=0, +SET renewing = 0, + renew_retry=0, not_after=?, updated_at=? WHERE id = ? @@ -191,18 +202,18 @@ func (q *Queries) UpdateCertAfterRenewal(ctx context.Context, arg UpdateCertAfte const updateRenewingState = `-- name: UpdateRenewingState :exec UPDATE certificates -SET renewing = ?, - renew_failed = ? +SET renewing = ?, + renew_retry = ? WHERE id = ? ` type UpdateRenewingStateParams struct { - Renewing bool `json:"renewing"` - RenewFailed bool `json:"renew_failed"` - ID int64 `json:"id"` + Renewing bool `json:"renewing"` + RenewRetry time.Time `json:"renew_retry"` + ID int64 `json:"id"` } func (q *Queries) UpdateRenewingState(ctx context.Context, arg UpdateRenewingStateParams) error { - _, err := q.db.ExecContext(ctx, updateRenewingState, arg.Renewing, arg.RenewFailed, arg.ID) + _, err := q.db.ExecContext(ctx, updateRenewingState, arg.Renewing, arg.RenewRetry, arg.ID) return err } diff --git a/database/migrations/20240920175046_retry_renewal.down.sql b/database/migrations/20240920175046_retry_renewal.down.sql new file mode 100644 index 0000000..e69de29 diff --git a/database/migrations/20240920175046_retry_renewal.up.sql b/database/migrations/20240920175046_retry_renewal.up.sql new file mode 100644 index 0000000..134be39 --- /dev/null +++ b/database/migrations/20240920175046_retry_renewal.up.sql @@ -0,0 +1,5 @@ +ALTER TABLE certificates + DROP COLUMN renew_failed; + +ALTER TABLE certificates + ADD COLUMN renew_retry DATETIME NOT NULL DEFAULT 0; diff --git a/database/models.go b/database/models.go index 4140fad..7286181 100644 --- a/database/models.go +++ b/database/models.go @@ -10,16 +10,16 @@ import ( ) type Certificate struct { - ID int64 `json:"id"` - Owner string `json:"owner"` - Dns sql.NullInt64 `json:"dns"` - AutoRenew bool `json:"auto_renew"` - Active bool `json:"active"` - Renewing bool `json:"renewing"` - RenewFailed bool `json:"renew_failed"` - NotAfter time.Time `json:"not_after"` - UpdatedAt time.Time `json:"updated_at"` - TempParent sql.NullInt64 `json:"temp_parent"` + ID int64 `json:"id"` + Owner string `json:"owner"` + Dns sql.NullInt64 `json:"dns"` + AutoRenew bool `json:"auto_renew"` + Active bool `json:"active"` + Renewing bool `json:"renewing"` + NotAfter time.Time `json:"not_after"` + UpdatedAt time.Time `json:"updated_at"` + TempParent sql.NullInt64 `json:"temp_parent"` + RenewRetry time.Time `json:"renew_retry"` } type CertificateDomain struct { diff --git a/database/queries/certificate.sql b/database/queries/certificate.sql index 45bb2f2..d1c978d 100644 --- a/database/queries/certificate.sql +++ b/database/queries/certificate.sql @@ -5,7 +5,7 @@ FROM certificates AS cert WHERE cert.active = 1 AND (cert.auto_renew = 1 OR cert.not_after IS NULL) AND cert.renewing = 0 - AND cert.renew_failed = 0 + AND DATETIME() > DATETIME(cert.renew_retry) AND (cert.not_after IS NULL OR DATETIME(cert.not_after, 'utc', '-30 days') < DATETIME()) ORDER BY cert.temp_parent, cert.not_after DESC NULLS FIRST LIMIT 1; @@ -15,7 +15,7 @@ SELECT cert.id, cert.auto_renew, cert.active, cert.renewing, - cert.renew_failed, + cert.renew_retry, cert.not_after, cert.updated_at, certificate_domains.domain @@ -24,14 +24,19 @@ FROM certificates AS cert -- name: UpdateRenewingState :exec UPDATE certificates -SET renewing = ?, - renew_failed = ? +SET renewing = ?, + renew_retry = ? +WHERE id = ?; + +-- name: SetRetryFlag :exec +UPDATE certificates +SET renew_retry = DATETIME('now', '+1 day') WHERE id = ?; -- name: UpdateCertAfterRenewal :exec UPDATE certificates -SET renewing = 0, - renew_failed=0, +SET renewing = 0, + renew_retry=0, not_after=?, updated_at=? WHERE id = ?; diff --git a/renewal/service.go b/renewal/service.go index 31655cb..db85085 100644 --- a/renewal/service.go +++ b/renewal/service.go @@ -399,13 +399,14 @@ func (s *Service) getPrivateKey(id int64) (*rsa.PrivateKey, error) { // certificate to the certDir directory. func (s *Service) renewCert(localData *localCertData) error { // database synchronous state - s.setRenewing(localData.id, true, false) + s.setRenewing(localData.id, true) Logger.Debug("No certificates to update") // run internal renewal code and log errors cert, certBytes, err := s.renewCertInternal(localData) if err != nil { - s.setRenewing(localData.id, false, true) + s.setRenewing(localData.id, false) + s.setRetry(localData.id) return fmt.Errorf("failed to renew cert %d: %w", localData.id, err) } @@ -501,14 +502,20 @@ func (s *Service) renewCertInternal(localData *localCertData) (*x509.Certificate // setRenewing sets the renewing and failed states in the database for a // specified certificate id. -func (s *Service) setRenewing(id int64, renewing, failed bool) { +func (s *Service) setRenewing(id int64, renewing bool) { err := s.db.UpdateRenewingState(context.Background(), database.UpdateRenewingStateParams{ - Renewing: renewing, - RenewFailed: failed, - ID: id, + Renewing: renewing, + ID: id, }) if err != nil { - Logger.Warn("Failed to set renewing/failed mode in database", "id", id, "err", err) + Logger.Warn("Failed to set renewing mode in database", "id", id, "err", err) + } +} + +func (s *Service) setRetry(id int64) { + err := s.db.SetRetryFlag(context.Background(), id) + if err != nil { + Logger.Warn("Failed to set retry time in database", "id", id, "err", err) } } diff --git a/renewal/service_test.go b/renewal/service_test.go index 6246bc9..2d71c70 100644 --- a/renewal/service_test.go +++ b/renewal/service_test.go @@ -170,7 +170,7 @@ func TestPebbleRenewal(t *testing.T) { _, err := db2.Exec("DELETE FROM certificate_domains") assert.NoError(t, err) - _, err = db2.Exec(`INSERT INTO certificates (owner, dns, auto_renew, active, renewing, renew_failed, not_after, updated_at) VALUES (1, 1, 1, 1, 0, 0, "2000-01-01 00:00:00+00:00", "2000-01-01 00:00:00+00:00")`) + _, err = db2.Exec(`INSERT INTO certificates (owner, dns, auto_renew, active, renewing, renew_retry, not_after, updated_at) VALUES (1, 1, 1, 1, 0, '2000-01-01 00:00:00+00:00', '2000-01-01 00:00:00+00:00', '2000-01-01 00:00:00+00:00');`) assert.NoError(t, err) for _, j := range i.domains { _, err = db2.Exec(`INSERT INTO certificate_domains (cert_id, domain) VALUES (1, ?)`, j) diff --git a/servers/api.go b/servers/api.go index 058eb17..7edab6d 100644 --- a/servers/api.go +++ b/servers/api.go @@ -25,14 +25,14 @@ type DomainStateValue struct { } type Certificate struct { - Id int64 `json:"id"` - AutoRenew bool `json:"auto_renew"` - Active bool `json:"active"` - Renewing bool `json:"renewing"` - RenewFailed bool `json:"renew_failed"` - NotAfter time.Time `json:"not_after"` - UpdatedAt time.Time `json:"updated_at"` - Domains []string `json:"domains"` + Id int64 `json:"id"` + AutoRenew bool `json:"auto_renew"` + Active bool `json:"active"` + Renewing bool `json:"renewing"` + RenewRetry time.Time `json:"renew_retry"` + NotAfter time.Time `json:"not_after"` + UpdatedAt time.Time `json:"updated_at"` + Domains []string `json:"domains"` } // NewApiServer creates and runs a http server containing all the API @@ -69,13 +69,13 @@ func NewApiServer(listen string, db *database.Queries, signer *mjwt.KeyStore, do // loop over query rows for _, row := range rows { c := Certificate{ - Id: row.ID, - AutoRenew: row.AutoRenew, - Active: row.Active, - Renewing: row.Renewing, - RenewFailed: row.RenewFailed, - NotAfter: row.NotAfter, - UpdatedAt: row.UpdatedAt, + Id: row.ID, + AutoRenew: row.AutoRenew, + Active: row.Active, + Renewing: row.Renewing, + RenewRetry: row.RenewRetry, + NotAfter: row.NotAfter, + UpdatedAt: row.UpdatedAt, } d := row.Domain