certificates.go 36 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775776777778779780781782783784785786787788789790791792793794795796797798799800801802803804805806807808809810811812813814815816817818819820821822823824825826827828829830831832833834835836837838839840841842843844845846847848849850851852853854855856857858859860861862863864865866867868869870871872873874875876877878879880881882883884885886887888889890891892893894895896897898899900901902903904905906907908909910911912913914915916917918919920921922923924925926927928929930931932933934935936937938939940941942943944945946947948949950951952953954955956957958959960961962963964965966967968969970971972973974975976977978979980981982983984985986987988989990991992
  1. package ca
  2. import (
  3. "bytes"
  4. "crypto"
  5. "crypto/ecdsa"
  6. "crypto/elliptic"
  7. cryptorand "crypto/rand"
  8. "crypto/rsa"
  9. "crypto/tls"
  10. "crypto/x509"
  11. "encoding/asn1"
  12. "encoding/pem"
  13. "fmt"
  14. "io"
  15. "io/ioutil"
  16. "os"
  17. "path/filepath"
  18. "time"
  19. cfcsr "github.com/cloudflare/cfssl/csr"
  20. "github.com/cloudflare/cfssl/helpers"
  21. "github.com/cloudflare/cfssl/initca"
  22. cflog "github.com/cloudflare/cfssl/log"
  23. cfsigner "github.com/cloudflare/cfssl/signer"
  24. "github.com/cloudflare/cfssl/signer/local"
  25. "github.com/docker/go-events"
  26. "github.com/docker/swarmkit/api"
  27. "github.com/docker/swarmkit/connectionbroker"
  28. "github.com/docker/swarmkit/ioutils"
  29. "github.com/opencontainers/go-digest"
  30. "github.com/pkg/errors"
  31. "golang.org/x/net/context"
  32. "google.golang.org/grpc"
  33. "google.golang.org/grpc/codes"
  34. "google.golang.org/grpc/credentials"
  35. )
  36. const (
  37. // Security Strength Equivalence
  38. //-----------------------------------
  39. //| ECC | DH/DSA/RSA |
  40. //| 256 | 3072 |
  41. //| 384 | 7680 |
  42. //-----------------------------------
  43. // RootKeySize is the default size of the root CA key
  44. // It would be ideal for the root key to use P-384, but in P-384 is not optimized in go yet :(
  45. RootKeySize = 256
  46. // RootKeyAlgo defines the default algorithm for the root CA Key
  47. RootKeyAlgo = "ecdsa"
  48. // PassphraseENVVar defines the environment variable to look for the
  49. // root CA private key material encryption key
  50. PassphraseENVVar = "SWARM_ROOT_CA_PASSPHRASE"
  51. // PassphraseENVVarPrev defines the alternate environment variable to look for the
  52. // root CA private key material encryption key. It can be used for seamless
  53. // KEK rotations.
  54. PassphraseENVVarPrev = "SWARM_ROOT_CA_PASSPHRASE_PREV"
  55. // RootCAExpiration represents the default expiration for the root CA in seconds (20 years)
  56. RootCAExpiration = "630720000s"
  57. // DefaultNodeCertExpiration represents the default expiration for node certificates (3 months)
  58. DefaultNodeCertExpiration = 2160 * time.Hour
  59. // CertBackdate represents the amount of time each certificate is backdated to try to avoid
  60. // clock drift issues.
  61. CertBackdate = 1 * time.Hour
  62. // CertLowerRotationRange represents the minimum fraction of time that we will wait when randomly
  63. // choosing our next certificate rotation
  64. CertLowerRotationRange = 0.5
  65. // CertUpperRotationRange represents the maximum fraction of time that we will wait when randomly
  66. // choosing our next certificate rotation
  67. CertUpperRotationRange = 0.8
  68. // MinNodeCertExpiration represents the minimum expiration for node certificates
  69. MinNodeCertExpiration = 1 * time.Hour
  70. )
  71. // BasicConstraintsOID is the ASN1 Object ID indicating a basic constraints extension
  72. var BasicConstraintsOID = asn1.ObjectIdentifier{2, 5, 29, 19}
  73. // A recoverableErr is a non-fatal error encountered signing a certificate,
  74. // which means that the certificate issuance may be retried at a later time.
  75. type recoverableErr struct {
  76. err error
  77. }
  78. func (r recoverableErr) Error() string {
  79. return r.err.Error()
  80. }
  81. // ErrNoLocalRootCA is an error type used to indicate that the local root CA
  82. // certificate file does not exist.
  83. var ErrNoLocalRootCA = errors.New("local root CA certificate does not exist")
  84. // ErrNoValidSigner is an error type used to indicate that our RootCA doesn't have the ability to
  85. // sign certificates.
  86. var ErrNoValidSigner = recoverableErr{err: errors.New("no valid signer found")}
  87. func init() {
  88. cflog.Level = 5
  89. }
  90. // CertPaths is a helper struct that keeps track of the paths of a
  91. // Cert and corresponding Key
  92. type CertPaths struct {
  93. Cert, Key string
  94. }
  95. // IssuerInfo contains the subject and public key of the issuer of a certificate
  96. type IssuerInfo struct {
  97. Subject []byte
  98. PublicKey []byte
  99. }
  100. // LocalSigner is a signer that can sign CSRs
  101. type LocalSigner struct {
  102. cfsigner.Signer
  103. // Key will only be used by the original manager to put the private
  104. // key-material in raft, no signing operations depend on it.
  105. Key []byte
  106. // Cert is one PEM encoded Certificate used as the signing CA. It must correspond to the key.
  107. Cert []byte
  108. // just cached parsed values for validation, etc.
  109. parsedCert *x509.Certificate
  110. cryptoSigner crypto.Signer
  111. }
  112. // RootCA is the representation of everything we need to sign certificates and/or to verify certificates
  113. //
  114. // RootCA.Cert: [CA cert1][CA cert2]
  115. // RootCA.Intermediates: [intermediate CA1][intermediate CA2][intermediate CA3]
  116. // RootCA.signer.Cert: [signing CA cert]
  117. // RootCA.signer.Key: [signing CA key]
  118. //
  119. // Requirements:
  120. //
  121. // - [signing CA key] must be the private key for [signing CA cert], and either both or none must be provided
  122. //
  123. // - [intermediate CA1] must have the same public key and subject as [signing CA cert], because otherwise when
  124. // appended to a leaf certificate, the intermediates will not form a chain (because [intermediate CA1] won't because
  125. // the signer of the leaf certificate)
  126. // - [intermediate CA1] must be signed by [intermediate CA2], which must be signed by [intermediate CA3]
  127. //
  128. // - When we issue a certificate, the intermediates will be appended so that the certificate looks like:
  129. // [leaf signed by signing CA cert][intermediate CA1][intermediate CA2][intermediate CA3]
  130. // - [leaf signed by signing CA cert][intermediate CA1][intermediate CA2][intermediate CA3] is guaranteed to form a
  131. // valid chain from [leaf signed by signing CA cert] to one of the root certs ([signing CA cert], [CA cert1], [CA cert2])
  132. // using zero or more of the intermediate certs ([intermediate CA1][intermediate CA2][intermediate CA3]) as intermediates
  133. //
  134. // Example 1: Simple root rotation
  135. // - Initial state:
  136. // - RootCA.Cert: [Root CA1 self-signed]
  137. // - RootCA.Intermediates: []
  138. // - RootCA.signer.Cert: [Root CA1 self-signed]
  139. // - Issued TLS cert: [leaf signed by Root CA1]
  140. //
  141. // - Intermediate state (during root rotation):
  142. // - RootCA.Cert: [Root CA1 self-signed]
  143. // - RootCA.Intermediates: [Root CA2 signed by Root CA1]
  144. // - RootCA.signer.Cert: [Root CA2 signed by Root CA1]
  145. // - Issued TLS cert: [leaf signed by Root CA2][Root CA2 signed by Root CA1]
  146. //
  147. // - Final state:
  148. // - RootCA.Cert: [Root CA2 self-signed]
  149. // - RootCA.Intermediates: []
  150. // - RootCA.signer.Cert: [Root CA2 self-signed]
  151. // - Issued TLS cert: [leaf signed by Root CA2]
  152. //
  153. type RootCA struct {
  154. // Certs contains a bundle of self-signed, PEM encoded certificates for the Root CA to be used
  155. // as the root of trust.
  156. Certs []byte
  157. // Intermediates contains a bundle of PEM encoded intermediate CA certificates to append to any
  158. // issued TLS (leaf) certificates. The first one must have the same public key and subject as the
  159. // signing root certificate, and the rest must form a chain, each one certifying the one above it,
  160. // as per RFC5246 section 7.4.2.
  161. Intermediates []byte
  162. // Pool is the root pool used to validate TLS certificates
  163. Pool *x509.CertPool
  164. // Digest of the serialized bytes of the certificate(s)
  165. Digest digest.Digest
  166. // This signer will be nil if the node doesn't have the appropriate key material
  167. signer *LocalSigner
  168. }
  169. // Signer is an accessor for the local signer that returns an error if this root cannot sign.
  170. func (rca *RootCA) Signer() (*LocalSigner, error) {
  171. if rca.Pool == nil || rca.signer == nil || len(rca.signer.Cert) == 0 || rca.signer.Signer == nil {
  172. return nil, ErrNoValidSigner
  173. }
  174. return rca.signer, nil
  175. }
  176. // IssueAndSaveNewCertificates generates a new key-pair, signs it with the local root-ca, and returns a
  177. // TLS certificate and the issuer information for the certificate.
  178. func (rca *RootCA) IssueAndSaveNewCertificates(kw KeyWriter, cn, ou, org string) (*tls.Certificate, *IssuerInfo, error) {
  179. csr, key, err := GenerateNewCSR()
  180. if err != nil {
  181. return nil, nil, errors.Wrap(err, "error when generating new node certs")
  182. }
  183. // Obtain a signed Certificate
  184. certChain, err := rca.ParseValidateAndSignCSR(csr, cn, ou, org)
  185. if err != nil {
  186. return nil, nil, errors.Wrap(err, "failed to sign node certificate")
  187. }
  188. signer, err := rca.Signer()
  189. if err != nil { // should never happen, since if ParseValidateAndSignCSR did not fail this root CA must have a signer
  190. return nil, nil, err
  191. }
  192. // Create a valid TLSKeyPair out of the PEM encoded private key and certificate
  193. tlsKeyPair, err := tls.X509KeyPair(certChain, key)
  194. if err != nil {
  195. return nil, nil, err
  196. }
  197. if err := kw.Write(NormalizePEMs(certChain), key, nil); err != nil {
  198. return nil, nil, err
  199. }
  200. return &tlsKeyPair, &IssuerInfo{
  201. PublicKey: signer.parsedCert.RawSubjectPublicKeyInfo,
  202. Subject: signer.parsedCert.RawSubject,
  203. }, nil
  204. }
  205. // RequestAndSaveNewCertificates gets new certificates issued, either by signing them locally if a signer is
  206. // available, or by requesting them from the remote server at remoteAddr. This function returns the TLS
  207. // certificate and the issuer information for the certificate.
  208. func (rca *RootCA) RequestAndSaveNewCertificates(ctx context.Context, kw KeyWriter, config CertificateRequestConfig) (*tls.Certificate, *IssuerInfo, error) {
  209. // Create a new key/pair and CSR
  210. csr, key, err := GenerateNewCSR()
  211. if err != nil {
  212. return nil, nil, errors.Wrap(err, "error when generating new node certs")
  213. }
  214. // Get the remote manager to issue a CA signed certificate for this node
  215. // Retry up to 5 times in case the manager we first try to contact isn't
  216. // responding properly (for example, it may have just been demoted).
  217. var signedCert []byte
  218. for i := 0; i != 5; i++ {
  219. signedCert, err = GetRemoteSignedCertificate(ctx, csr, rca.Pool, config)
  220. if err == nil {
  221. break
  222. }
  223. // If the first attempt fails, we should try a remote
  224. // connection. The local node may be a manager that was
  225. // demoted, so the local connection (which is preferred) may
  226. // not work. If we are successful in renewing the certificate,
  227. // the local connection will not be returned by the connection
  228. // broker anymore.
  229. config.ForceRemote = true
  230. // Wait a moment, in case a leader election was taking place.
  231. select {
  232. case <-time.After(config.RetryInterval):
  233. case <-ctx.Done():
  234. return nil, nil, ctx.Err()
  235. }
  236. }
  237. if err != nil {
  238. return nil, nil, err
  239. }
  240. // Доверяй, но проверяй.
  241. // Before we overwrite our local key + certificate, let's make sure the server gave us one that is valid
  242. // Create an X509Cert so we can .Verify()
  243. // Check to see if this certificate was signed by our CA, and isn't expired
  244. parsedCerts, chains, err := ValidateCertChain(rca.Pool, signedCert, false)
  245. if err != nil {
  246. return nil, nil, err
  247. }
  248. // ValidateChain, if successful, will always return at least 1 parsed cert and at least 1 chain containing
  249. // at least 2 certificates: the leaf and the root.
  250. leafCert := parsedCerts[0]
  251. issuer := chains[0][1]
  252. // Create a valid TLSKeyPair out of the PEM encoded private key and certificate
  253. tlsKeyPair, err := tls.X509KeyPair(signedCert, key)
  254. if err != nil {
  255. return nil, nil, err
  256. }
  257. var kekUpdate *KEKData
  258. for i := 0; i < 5; i++ {
  259. // ValidateCertChain will always return at least 1 cert, so indexing at 0 is safe
  260. kekUpdate, err = rca.getKEKUpdate(ctx, leafCert, tlsKeyPair, config)
  261. if err == nil {
  262. break
  263. }
  264. config.ForceRemote = true
  265. // Wait a moment, in case a leader election was taking place.
  266. select {
  267. case <-time.After(config.RetryInterval):
  268. case <-ctx.Done():
  269. return nil, nil, ctx.Err()
  270. }
  271. }
  272. if err != nil {
  273. return nil, nil, err
  274. }
  275. if err := kw.Write(NormalizePEMs(signedCert), key, kekUpdate); err != nil {
  276. return nil, nil, err
  277. }
  278. return &tlsKeyPair, &IssuerInfo{
  279. PublicKey: issuer.RawSubjectPublicKeyInfo,
  280. Subject: issuer.RawSubject,
  281. }, nil
  282. }
  283. func (rca *RootCA) getKEKUpdate(ctx context.Context, leafCert *x509.Certificate, keypair tls.Certificate, config CertificateRequestConfig) (*KEKData, error) {
  284. var managerRole bool
  285. for _, ou := range leafCert.Subject.OrganizationalUnit {
  286. if ou == ManagerRole {
  287. managerRole = true
  288. break
  289. }
  290. }
  291. if managerRole {
  292. mtlsCreds := credentials.NewTLS(&tls.Config{ServerName: CARole, RootCAs: rca.Pool, Certificates: []tls.Certificate{keypair}})
  293. conn, err := getGRPCConnection(mtlsCreds, config.ConnBroker, config.ForceRemote)
  294. if err != nil {
  295. return nil, err
  296. }
  297. client := api.NewCAClient(conn.ClientConn)
  298. ctx, cancel := context.WithTimeout(ctx, 5*time.Second)
  299. defer cancel()
  300. response, err := client.GetUnlockKey(ctx, &api.GetUnlockKeyRequest{})
  301. if err != nil {
  302. if grpc.Code(err) == codes.Unimplemented { // if the server does not support keks, return as if no encryption key was specified
  303. conn.Close(true)
  304. return &KEKData{}, nil
  305. }
  306. conn.Close(false)
  307. return nil, err
  308. }
  309. conn.Close(true)
  310. return &KEKData{KEK: response.UnlockKey, Version: response.Version.Index}, nil
  311. }
  312. // If this is a worker, set to never encrypt. We always want to set to the lock key to nil,
  313. // in case this was a manager that was demoted to a worker.
  314. return &KEKData{}, nil
  315. }
  316. // PrepareCSR creates a CFSSL Sign Request based on the given raw CSR and
  317. // overrides the Subject and Hosts with the given extra args.
  318. func PrepareCSR(csrBytes []byte, cn, ou, org string) cfsigner.SignRequest {
  319. // All managers get added the subject-alt-name of CA, so they can be
  320. // used for cert issuance.
  321. hosts := []string{ou, cn}
  322. if ou == ManagerRole {
  323. hosts = append(hosts, CARole)
  324. }
  325. return cfsigner.SignRequest{
  326. Request: string(csrBytes),
  327. // OU is used for Authentication of the node type. The CN has the random
  328. // node ID.
  329. Subject: &cfsigner.Subject{CN: cn, Names: []cfcsr.Name{{OU: ou, O: org}}},
  330. // Adding ou as DNS alt name, so clients can connect to ManagerRole and CARole
  331. Hosts: hosts,
  332. }
  333. }
  334. // ParseValidateAndSignCSR returns a signed certificate from a particular rootCA and a CSR.
  335. func (rca *RootCA) ParseValidateAndSignCSR(csrBytes []byte, cn, ou, org string) ([]byte, error) {
  336. signRequest := PrepareCSR(csrBytes, cn, ou, org)
  337. signer, err := rca.Signer()
  338. if err != nil {
  339. return nil, err
  340. }
  341. cert, err := signer.Sign(signRequest)
  342. if err != nil {
  343. return nil, errors.Wrap(err, "failed to sign node certificate")
  344. }
  345. return append(cert, rca.Intermediates...), nil
  346. }
  347. // CrossSignCACertificate takes a CA root certificate and generates an intermediate CA from it signed with the current root signer
  348. func (rca *RootCA) CrossSignCACertificate(otherCAPEM []byte) ([]byte, error) {
  349. signer, err := rca.Signer()
  350. if err != nil {
  351. return nil, err
  352. }
  353. // create a new cert with exactly the same parameters, including the public key and exact NotBefore and NotAfter
  354. template, err := helpers.ParseCertificatePEM(otherCAPEM)
  355. if err != nil {
  356. return nil, errors.New("could not parse new CA certificate")
  357. }
  358. if !template.IsCA {
  359. return nil, errors.New("certificate not a CA")
  360. }
  361. template.SignatureAlgorithm = signer.parsedCert.SignatureAlgorithm // make sure we can sign with the signer key
  362. derBytes, err := x509.CreateCertificate(cryptorand.Reader, template, signer.parsedCert, template.PublicKey, signer.cryptoSigner)
  363. if err != nil {
  364. return nil, errors.Wrap(err, "could not cross-sign new CA certificate using old CA material")
  365. }
  366. return pem.EncodeToMemory(&pem.Block{
  367. Type: "CERTIFICATE",
  368. Bytes: derBytes,
  369. }), nil
  370. }
  371. func validateSignatureAlgorithm(cert *x509.Certificate) error {
  372. switch cert.SignatureAlgorithm {
  373. case x509.SHA256WithRSA, x509.SHA384WithRSA, x509.SHA512WithRSA, x509.ECDSAWithSHA256, x509.ECDSAWithSHA384, x509.ECDSAWithSHA512:
  374. return nil
  375. default:
  376. return fmt.Errorf("unsupported signature algorithm: %s", cert.SignatureAlgorithm.String())
  377. }
  378. }
  379. // NewRootCA creates a new RootCA object from unparsed PEM cert bundle and key byte
  380. // slices. key may be nil, and in this case NewRootCA will return a RootCA
  381. // without a signer.
  382. func NewRootCA(rootCertBytes, signCertBytes, signKeyBytes []byte, certExpiry time.Duration, intermediates []byte) (RootCA, error) {
  383. // Parse all the certificates in the cert bundle
  384. parsedCerts, err := helpers.ParseCertificatesPEM(rootCertBytes)
  385. if err != nil {
  386. return RootCA{}, errors.Wrap(err, "invalid root certificates")
  387. }
  388. // Check to see if we have at least one valid cert
  389. if len(parsedCerts) < 1 {
  390. return RootCA{}, errors.New("no valid root CA certificates found")
  391. }
  392. // Create a Pool with all of the certificates found
  393. pool := x509.NewCertPool()
  394. for _, cert := range parsedCerts {
  395. if err := validateSignatureAlgorithm(cert); err != nil {
  396. return RootCA{}, err
  397. }
  398. // Check to see if all of the certificates are valid, self-signed root CA certs
  399. selfpool := x509.NewCertPool()
  400. selfpool.AddCert(cert)
  401. if _, err := cert.Verify(x509.VerifyOptions{Roots: selfpool}); err != nil {
  402. return RootCA{}, errors.Wrap(err, "error while validating Root CA Certificate")
  403. }
  404. pool.AddCert(cert)
  405. }
  406. // Calculate the digest for our Root CA bundle
  407. digest := digest.FromBytes(rootCertBytes)
  408. // The intermediates supplied must be able to chain up to the root certificates, so that when they are appended to
  409. // a leaf certificate, the leaf certificate can be validated through the intermediates to the root certificates.
  410. var intermediatePool *x509.CertPool
  411. var parsedIntermediates []*x509.Certificate
  412. if len(intermediates) > 0 {
  413. parsedIntermediates, _, err = ValidateCertChain(pool, intermediates, false)
  414. if err != nil {
  415. return RootCA{}, errors.Wrap(err, "invalid intermediate chain")
  416. }
  417. intermediatePool = x509.NewCertPool()
  418. for _, cert := range parsedIntermediates {
  419. intermediatePool.AddCert(cert)
  420. }
  421. }
  422. var localSigner *LocalSigner
  423. if len(signKeyBytes) != 0 || len(signCertBytes) != 0 {
  424. localSigner, err = newLocalSigner(signKeyBytes, signCertBytes, certExpiry, pool, intermediatePool)
  425. if err != nil {
  426. return RootCA{}, err
  427. }
  428. // If a signer is provided and there are intermediates, then either the first intermediate would be the signer CA
  429. // certificate (in which case it'd have the same subject and public key), or it would be a cross-signed
  430. // intermediate with the same subject and public key as our signing CA certificate (which could be either an
  431. // intermediate cert or a self-signed root cert).
  432. if len(parsedIntermediates) > 0 && (!bytes.Equal(parsedIntermediates[0].RawSubject, localSigner.parsedCert.RawSubject) ||
  433. !bytes.Equal(parsedIntermediates[0].RawSubjectPublicKeyInfo, localSigner.parsedCert.RawSubjectPublicKeyInfo)) {
  434. return RootCA{}, errors.New(
  435. "invalid intermediate chain - the first intermediate must have the same subject and public key as the signing cert")
  436. }
  437. }
  438. return RootCA{signer: localSigner, Intermediates: intermediates, Digest: digest, Certs: rootCertBytes, Pool: pool}, nil
  439. }
  440. // ValidateCertChain checks checks that the certificates provided chain up to the root pool provided. In addition
  441. // it also enforces that every cert in the bundle certificates form a chain, each one certifying the one above,
  442. // as per RFC5246 section 7.4.2, and that every certificate (whether or not it is necessary to form a chain to the root
  443. // pool) is currently valid and not yet expired (unless allowExpiry is set to true).
  444. // This is additional validation not required by go's Certificate.Verify (which allows invalid certs in the
  445. // intermediate pool), because this function is intended to be used when reading certs from untrusted locations such as
  446. // from disk or over a network when a CSR is signed, so it is extra pedantic.
  447. // This function always returns all the parsed certificates in the bundle in order, which means there will always be
  448. // at least 1 certificate if there is no error, and the valid chains found by Certificate.Verify
  449. func ValidateCertChain(rootPool *x509.CertPool, certs []byte, allowExpired bool) ([]*x509.Certificate, [][]*x509.Certificate, error) {
  450. // Parse all the certificates in the cert bundle
  451. parsedCerts, err := helpers.ParseCertificatesPEM(certs)
  452. if err != nil {
  453. return nil, nil, err
  454. }
  455. if len(parsedCerts) == 0 {
  456. return nil, nil, errors.New("no certificates to validate")
  457. }
  458. now := time.Now()
  459. // ensure that they form a chain, each one being signed by the one after it
  460. var intermediatePool *x509.CertPool
  461. for i, cert := range parsedCerts {
  462. // Manual expiry validation because we want more information on which certificate in the chain is expired, and
  463. // because this is an easier way to allow expired certs.
  464. if now.Before(cert.NotBefore) {
  465. return nil, nil, errors.Wrapf(
  466. x509.CertificateInvalidError{
  467. Cert: cert,
  468. Reason: x509.Expired,
  469. },
  470. "certificate (%d - %s) not valid before %s, and it is currently %s",
  471. i+1, cert.Subject.CommonName, cert.NotBefore.UTC().Format(time.RFC1123), now.Format(time.RFC1123))
  472. }
  473. if !allowExpired && now.After(cert.NotAfter) {
  474. return nil, nil, errors.Wrapf(
  475. x509.CertificateInvalidError{
  476. Cert: cert,
  477. Reason: x509.Expired,
  478. },
  479. "certificate (%d - %s) not valid after %s, and it is currently %s",
  480. i+1, cert.Subject.CommonName, cert.NotAfter.UTC().Format(time.RFC1123), now.Format(time.RFC1123))
  481. }
  482. if i > 0 {
  483. // check that the previous cert was signed by this cert
  484. prevCert := parsedCerts[i-1]
  485. if err := prevCert.CheckSignatureFrom(cert); err != nil {
  486. return nil, nil, errors.Wrapf(err, "certificates do not form a chain: (%d - %s) is not signed by (%d - %s)",
  487. i, prevCert.Subject.CommonName, i+1, cert.Subject.CommonName)
  488. }
  489. if intermediatePool == nil {
  490. intermediatePool = x509.NewCertPool()
  491. }
  492. intermediatePool.AddCert(cert)
  493. }
  494. }
  495. verifyOpts := x509.VerifyOptions{
  496. Roots: rootPool,
  497. Intermediates: intermediatePool,
  498. CurrentTime: now,
  499. }
  500. var chains [][]*x509.Certificate
  501. // If we accept expired certs, try to build a valid cert chain using some subset of the certs. We start off using the
  502. // first certificate's NotAfter as the current time, thus ensuring that the first cert is not expired. If the chain
  503. // still fails to validate due to expiry issues, continue iterating over the rest of the certs.
  504. // If any of the other certs has an earlier NotAfter time, use that time as the current time instead. This insures that
  505. // particular cert, and any that came before it, are not expired. Note that the root that the certs chain up to
  506. // should also not be expired at that "current" time.
  507. if allowExpired {
  508. verifyOpts.CurrentTime = parsedCerts[0].NotAfter.Add(time.Hour)
  509. for _, cert := range parsedCerts {
  510. if !cert.NotAfter.Before(verifyOpts.CurrentTime) {
  511. continue
  512. }
  513. verifyOpts.CurrentTime = cert.NotAfter
  514. chains, err = parsedCerts[0].Verify(verifyOpts)
  515. if err == nil {
  516. return parsedCerts, chains, nil
  517. }
  518. }
  519. if invalid, ok := err.(x509.CertificateInvalidError); ok && invalid.Reason == x509.Expired {
  520. return nil, nil, errors.New("there is no time span for which all of the certificates, including a root, are valid")
  521. }
  522. return nil, nil, err
  523. }
  524. chains, err = parsedCerts[0].Verify(verifyOpts)
  525. if err != nil {
  526. return nil, nil, err
  527. }
  528. return parsedCerts, chains, nil
  529. }
  530. // newLocalSigner validates the signing cert and signing key to create a local signer, which accepts a crypto signer and a cert
  531. func newLocalSigner(keyBytes, certBytes []byte, certExpiry time.Duration, rootPool, intermediatePool *x509.CertPool) (*LocalSigner, error) {
  532. if len(keyBytes) == 0 || len(certBytes) == 0 {
  533. return nil, errors.New("must provide both a signing key and a signing cert, or neither")
  534. }
  535. parsedCerts, err := helpers.ParseCertificatesPEM(certBytes)
  536. if err != nil {
  537. return nil, errors.Wrap(err, "invalid signing CA cert")
  538. }
  539. if len(parsedCerts) == 0 {
  540. return nil, errors.New("no valid signing CA certificates found")
  541. }
  542. if err := validateSignatureAlgorithm(parsedCerts[0]); err != nil {
  543. return nil, err
  544. }
  545. opts := x509.VerifyOptions{
  546. Roots: rootPool,
  547. Intermediates: intermediatePool,
  548. }
  549. if _, err := parsedCerts[0].Verify(opts); err != nil {
  550. return nil, errors.Wrap(err, "error while validating signing CA certificate against roots and intermediates")
  551. }
  552. var (
  553. passphraseStr string
  554. passphrase, passphrasePrev []byte
  555. priv crypto.Signer
  556. )
  557. // Attempt two distinct passphrases, so we can do a hitless passphrase rotation
  558. if passphraseStr = os.Getenv(PassphraseENVVar); passphraseStr != "" {
  559. passphrase = []byte(passphraseStr)
  560. }
  561. if p := os.Getenv(PassphraseENVVarPrev); p != "" {
  562. passphrasePrev = []byte(p)
  563. }
  564. // Attempt to decrypt the current private-key with the passphrases provided
  565. priv, err = helpers.ParsePrivateKeyPEMWithPassword(keyBytes, passphrase)
  566. if err != nil {
  567. priv, err = helpers.ParsePrivateKeyPEMWithPassword(keyBytes, passphrasePrev)
  568. if err != nil {
  569. return nil, errors.Wrap(err, "malformed private key")
  570. }
  571. }
  572. // We will always use the first certificate inside of the root bundle as the active one
  573. if err := ensureCertKeyMatch(parsedCerts[0], priv.Public()); err != nil {
  574. return nil, err
  575. }
  576. signer, err := local.NewSigner(priv, parsedCerts[0], cfsigner.DefaultSigAlgo(priv), SigningPolicy(certExpiry))
  577. if err != nil {
  578. return nil, err
  579. }
  580. // If the key was loaded from disk unencrypted, but there is a passphrase set,
  581. // ensure it is encrypted, so it doesn't hit raft in plain-text
  582. // we don't have to check for nil, because if we couldn't pem-decode the bytes, then parsing above would have failed
  583. keyBlock, _ := pem.Decode(keyBytes)
  584. if passphraseStr != "" && !x509.IsEncryptedPEMBlock(keyBlock) {
  585. keyBytes, err = EncryptECPrivateKey(keyBytes, passphraseStr)
  586. if err != nil {
  587. return nil, errors.Wrap(err, "unable to encrypt signing CA key material")
  588. }
  589. }
  590. return &LocalSigner{Cert: certBytes, Key: keyBytes, Signer: signer, parsedCert: parsedCerts[0], cryptoSigner: priv}, nil
  591. }
  592. func ensureCertKeyMatch(cert *x509.Certificate, key crypto.PublicKey) error {
  593. switch certPub := cert.PublicKey.(type) {
  594. case *rsa.PublicKey:
  595. if certPub.N.BitLen() < 2048 || certPub.E == 1 {
  596. return errors.New("unsupported RSA key parameters")
  597. }
  598. rsaKey, ok := key.(*rsa.PublicKey)
  599. if ok && certPub.E == rsaKey.E && certPub.N.Cmp(rsaKey.N) == 0 {
  600. return nil
  601. }
  602. case *ecdsa.PublicKey:
  603. switch certPub.Curve {
  604. case elliptic.P256(), elliptic.P384(), elliptic.P521():
  605. break
  606. default:
  607. return errors.New("unsupported ECDSA key parameters")
  608. }
  609. ecKey, ok := key.(*ecdsa.PublicKey)
  610. if ok && certPub.X.Cmp(ecKey.X) == 0 && certPub.Y.Cmp(ecKey.Y) == 0 {
  611. return nil
  612. }
  613. default:
  614. return errors.New("unknown or unsupported certificate public key algorithm")
  615. }
  616. return errors.New("certificate key mismatch")
  617. }
  618. // GetLocalRootCA validates if the contents of the file are a valid self-signed
  619. // CA certificate, and returns the PEM-encoded Certificate if so
  620. func GetLocalRootCA(paths CertPaths) (RootCA, error) {
  621. // Check if we have a Certificate file
  622. cert, err := ioutil.ReadFile(paths.Cert)
  623. if err != nil {
  624. if os.IsNotExist(err) {
  625. err = ErrNoLocalRootCA
  626. }
  627. return RootCA{}, err
  628. }
  629. signingCert := cert
  630. key, err := ioutil.ReadFile(paths.Key)
  631. if err != nil {
  632. if !os.IsNotExist(err) {
  633. return RootCA{}, err
  634. }
  635. // There may not be a local key. It's okay to pass in a nil
  636. // key. We'll get a root CA without a signer.
  637. key = nil
  638. signingCert = nil
  639. }
  640. return NewRootCA(cert, signingCert, key, DefaultNodeCertExpiration, nil)
  641. }
  642. func getGRPCConnection(creds credentials.TransportCredentials, connBroker *connectionbroker.Broker, forceRemote bool) (*connectionbroker.Conn, error) {
  643. dialOpts := []grpc.DialOption{
  644. grpc.WithTransportCredentials(creds),
  645. grpc.WithTimeout(5 * time.Second),
  646. grpc.WithBackoffMaxDelay(5 * time.Second),
  647. }
  648. if forceRemote {
  649. return connBroker.SelectRemote(dialOpts...)
  650. }
  651. return connBroker.Select(dialOpts...)
  652. }
  653. // GetRemoteCA returns the remote endpoint's CA certificate bundle
  654. func GetRemoteCA(ctx context.Context, d digest.Digest, connBroker *connectionbroker.Broker) (RootCA, error) {
  655. // This TLS Config is intentionally using InsecureSkipVerify. We use the
  656. // digest instead to check the integrity of the CA certificate.
  657. insecureCreds := credentials.NewTLS(&tls.Config{InsecureSkipVerify: true})
  658. conn, err := getGRPCConnection(insecureCreds, connBroker, false)
  659. if err != nil {
  660. return RootCA{}, err
  661. }
  662. client := api.NewCAClient(conn.ClientConn)
  663. ctx, cancel := context.WithTimeout(ctx, 5*time.Second)
  664. defer cancel()
  665. defer func() {
  666. conn.Close(err == nil)
  667. }()
  668. response, err := client.GetRootCACertificate(ctx, &api.GetRootCACertificateRequest{})
  669. if err != nil {
  670. return RootCA{}, err
  671. }
  672. // If a bundle of certificates are provided, the digest covers the entire bundle and not just
  673. // one of the certificates in the bundle. Otherwise, a node can be MITMed while joining if
  674. // the MITM CA provides a single certificate which matches the digest, and providing arbitrary
  675. // other non-verified root certs that the manager certificate actually chains up to.
  676. if d != "" {
  677. verifier := d.Verifier()
  678. if err != nil {
  679. return RootCA{}, errors.Wrap(err, "unexpected error getting digest verifier")
  680. }
  681. io.Copy(verifier, bytes.NewReader(response.Certificate))
  682. if !verifier.Verified() {
  683. return RootCA{}, errors.Errorf("remote CA does not match fingerprint. Expected: %s", d.Hex())
  684. }
  685. }
  686. // NewRootCA will validate that the certificates are otherwise valid and create a RootCA object.
  687. // Since there is no key, the certificate expiry does not matter and will not be used.
  688. return NewRootCA(response.Certificate, nil, nil, DefaultNodeCertExpiration, nil)
  689. }
  690. // CreateRootCA creates a Certificate authority for a new Swarm Cluster, potentially
  691. // overwriting any existing CAs.
  692. func CreateRootCA(rootCN string) (RootCA, error) {
  693. // Create a simple CSR for the CA using the default CA validator and policy
  694. req := cfcsr.CertificateRequest{
  695. CN: rootCN,
  696. KeyRequest: &cfcsr.BasicKeyRequest{A: RootKeyAlgo, S: RootKeySize},
  697. CA: &cfcsr.CAConfig{Expiry: RootCAExpiration},
  698. }
  699. // Generate the CA and get the certificate and private key
  700. cert, _, key, err := initca.New(&req)
  701. if err != nil {
  702. return RootCA{}, err
  703. }
  704. rootCA, err := NewRootCA(cert, cert, key, DefaultNodeCertExpiration, nil)
  705. if err != nil {
  706. return RootCA{}, err
  707. }
  708. return rootCA, nil
  709. }
  710. // GetRemoteSignedCertificate submits a CSR to a remote CA server address,
  711. // and that is part of a CA identified by a specific certificate pool.
  712. func GetRemoteSignedCertificate(ctx context.Context, csr []byte, rootCAPool *x509.CertPool, config CertificateRequestConfig) ([]byte, error) {
  713. if rootCAPool == nil {
  714. return nil, errors.New("valid root CA pool required")
  715. }
  716. creds := config.Credentials
  717. if creds == nil {
  718. // This is our only non-MTLS request, and it happens when we are boostraping our TLS certs
  719. // We're using CARole as server name, so an external CA doesn't also have to have ManagerRole in the cert SANs
  720. creds = credentials.NewTLS(&tls.Config{ServerName: CARole, RootCAs: rootCAPool})
  721. }
  722. conn, err := getGRPCConnection(creds, config.ConnBroker, config.ForceRemote)
  723. if err != nil {
  724. return nil, err
  725. }
  726. // Create a CAClient to retrieve a new Certificate
  727. caClient := api.NewNodeCAClient(conn.ClientConn)
  728. issueCtx, issueCancel := context.WithTimeout(ctx, 5*time.Second)
  729. defer issueCancel()
  730. // Send the Request and retrieve the request token
  731. issueRequest := &api.IssueNodeCertificateRequest{CSR: csr, Token: config.Token, Availability: config.Availability}
  732. issueResponse, err := caClient.IssueNodeCertificate(issueCtx, issueRequest)
  733. if err != nil {
  734. conn.Close(false)
  735. return nil, err
  736. }
  737. statusRequest := &api.NodeCertificateStatusRequest{NodeID: issueResponse.NodeID}
  738. expBackoff := events.NewExponentialBackoff(events.ExponentialBackoffConfig{
  739. Base: time.Second,
  740. Factor: time.Second,
  741. Max: 30 * time.Second,
  742. })
  743. // Exponential backoff with Max of 30 seconds to wait for a new retry
  744. for {
  745. timeout := 5 * time.Second
  746. if config.NodeCertificateStatusRequestTimeout > 0 {
  747. timeout = config.NodeCertificateStatusRequestTimeout
  748. }
  749. // Send the Request and retrieve the certificate
  750. stateCtx, cancel := context.WithTimeout(ctx, timeout)
  751. defer cancel()
  752. statusResponse, err := caClient.NodeCertificateStatus(stateCtx, statusRequest)
  753. switch {
  754. case err != nil && grpc.Code(err) != codes.DeadlineExceeded:
  755. conn.Close(false)
  756. // Because IssueNodeCertificate succeeded, if this call failed likely it is due to an issue with this
  757. // particular connection, so we need to get another. We should try a remote connection - the local node
  758. // may be a manager that was demoted, so the local connection (which is preferred) may not work.
  759. config.ForceRemote = true
  760. conn, err = getGRPCConnection(creds, config.ConnBroker, config.ForceRemote)
  761. if err != nil {
  762. return nil, err
  763. }
  764. caClient = api.NewNodeCAClient(conn.ClientConn)
  765. // If there was no deadline exceeded error, and the certificate was issued, return
  766. case err == nil && statusResponse.Status.State == api.IssuanceStateIssued:
  767. if statusResponse.Certificate == nil {
  768. conn.Close(false)
  769. return nil, errors.New("no certificate in CertificateStatus response")
  770. }
  771. // The certificate in the response must match the CSR
  772. // we submitted. If we are getting a response for a
  773. // certificate that was previously issued, we need to
  774. // retry until the certificate gets updated per our
  775. // current request.
  776. if bytes.Equal(statusResponse.Certificate.CSR, csr) {
  777. conn.Close(true)
  778. return statusResponse.Certificate.Certificate, nil
  779. }
  780. }
  781. // If NodeCertificateStatus timed out, we're still pending, the issuance failed, or
  782. // the state is unknown let's continue trying after an exponential backoff
  783. expBackoff.Failure(nil, nil)
  784. select {
  785. case <-ctx.Done():
  786. conn.Close(true)
  787. return nil, err
  788. case <-time.After(expBackoff.Proceed(nil)):
  789. }
  790. }
  791. }
  792. // readCertValidity returns the certificate issue and expiration time
  793. func readCertValidity(kr KeyReader) (time.Time, time.Time, error) {
  794. var zeroTime time.Time
  795. // Read the Cert
  796. cert, _, err := kr.Read()
  797. if err != nil {
  798. return zeroTime, zeroTime, err
  799. }
  800. // Create an x509 certificate out of the contents on disk
  801. certBlock, _ := pem.Decode(cert)
  802. if certBlock == nil {
  803. return zeroTime, zeroTime, errors.New("failed to decode certificate block")
  804. }
  805. X509Cert, err := x509.ParseCertificate(certBlock.Bytes)
  806. if err != nil {
  807. return zeroTime, zeroTime, err
  808. }
  809. return X509Cert.NotBefore, X509Cert.NotAfter, nil
  810. }
  811. // SaveRootCA saves a RootCA object to disk
  812. func SaveRootCA(rootCA RootCA, paths CertPaths) error {
  813. // Make sure the necessary dirs exist and they are writable
  814. err := os.MkdirAll(filepath.Dir(paths.Cert), 0755)
  815. if err != nil {
  816. return err
  817. }
  818. // If the root certificate got returned successfully, save the rootCA to disk.
  819. return ioutils.AtomicWriteFile(paths.Cert, rootCA.Certs, 0644)
  820. }
  821. // GenerateNewCSR returns a newly generated key and CSR signed with said key
  822. func GenerateNewCSR() ([]byte, []byte, error) {
  823. req := &cfcsr.CertificateRequest{
  824. KeyRequest: cfcsr.NewBasicKeyRequest(),
  825. }
  826. return cfcsr.ParseRequest(req)
  827. }
  828. // EncryptECPrivateKey receives a PEM encoded private key and returns an encrypted
  829. // AES256 version using a passphrase
  830. // TODO: Make this method generic to handle RSA keys
  831. func EncryptECPrivateKey(key []byte, passphraseStr string) ([]byte, error) {
  832. passphrase := []byte(passphraseStr)
  833. cipherType := x509.PEMCipherAES256
  834. keyBlock, _ := pem.Decode(key)
  835. if keyBlock == nil {
  836. // This RootCA does not have a valid signer.
  837. return nil, errors.New("error while decoding PEM key")
  838. }
  839. encryptedPEMBlock, err := x509.EncryptPEMBlock(cryptorand.Reader,
  840. "EC PRIVATE KEY",
  841. keyBlock.Bytes,
  842. passphrase,
  843. cipherType)
  844. if err != nil {
  845. return nil, err
  846. }
  847. if encryptedPEMBlock.Headers == nil {
  848. return nil, errors.New("unable to encrypt key - invalid PEM file produced")
  849. }
  850. return pem.EncodeToMemory(encryptedPEMBlock), nil
  851. }
  852. // NormalizePEMs takes a bundle of PEM-encoded certificates in a certificate bundle,
  853. // decodes them, removes headers, and re-encodes them to make sure that they have
  854. // consistent whitespace. Note that this is intended to normalize x509 certificates
  855. // in PEM format, hence the stripping out of headers.
  856. func NormalizePEMs(certs []byte) []byte {
  857. var (
  858. results []byte
  859. pemBlock *pem.Block
  860. )
  861. for {
  862. pemBlock, certs = pem.Decode(bytes.TrimSpace(certs))
  863. if pemBlock == nil {
  864. return results
  865. }
  866. pemBlock.Headers = nil
  867. results = append(results, pem.EncodeToMemory(pemBlock)...)
  868. }
  869. }