Merge pull request #120 from darkowlzz/metadata-update-verify

Automate OLM metadata update and verify

Author: darkowlzz

Makefile

@@ -76,8 +76,15 @@ metadata-zip:
 		deploy/olm/storageos/storageosupgrade.crd.yaml \
 		deploy/olm/csv-rhel/storageos.clusterserviceversion.yaml
 
+metadata-update:
+	# Update all the metadata files in-place.
+	bash scripts/metadata-checker/update-metadata-files.sh
+
 # Lint the OLM metadata bundle.
 olm-lint:
+	# Generate metadata files and verify all the metadata files are up-to-date.
+	bash scripts/metadata-checker/metadata-diff-checker.sh
+	# Verify the OLM metada using operator-courier.
 	docker run -it --rm -v $(PWD)/deploy/olm/storageos/:/storageos \
 		python:3 bash -c "pip install operator-courier && operator-courier verify --ui_validate_io /storageos"
 

README.md

@@ -23,10 +23,10 @@ for more information.
 
 ## Setup/Development
 
-1. Install [operator-sdk](https://github.com/operator-framework/operator-sdk/tree/master#quick-start).
-2. Run `operator-sdk generate k8s` if there's a change in api type.
-3. Build operator container with `operator-sdk build storageos/cluster-operator:<tag>`
-4. Apply the manifests in `deploy/` to install the operator
+1. Build operator container image with `make image/cluster-operator`. Publish or
+  copy this container image to an existing k8s cluster to make it available
+  for use within the cluster.
+2. Apply the manifests in `deploy/` to install the operator
    * Apply `namespace.yaml` to create the `storageos-operator` namespace.
    * Apply `service_account.yaml`, `role.yaml` and `role_binding.yaml` to create
     a service account and to grant all the permissions.
@@ -53,7 +53,11 @@ make image/cluster-operator OPERATOR_IMAGE=storageos/cluster-operator:test
 
 This builds all the components and copies the binaries into the same container.
 
-After creating a resource, query the resource:
+For any changes related to Operator Lifecycle Manager(OLM), update
+`deploy/storageos-operators.configmap.yaml` and run `make metadata-update` to
+automatically update all the CRD, CSV and package files.
+
+After creating a StorageOSCluster resource, query the resource:
 
 ```bash
 $ kubectl get storageoscluster

deploy/olm/README.md

@@ -0,0 +1,21 @@
+## csv-rhel/
+
+This directory contains all the CSV files for rhel releases.
+
+## storageos/
+
+This directory contains all the CSV files for community operator releases. It
+also contains CRD and package files. This directory is also used to create a
+diff and submit a PR to the community operator repo for publishing new
+release. rhel releases uses the CRD and package files from this directory.
+
+## olm.sh
+
+This script contains helper functions to setup OLM in a cluster, install
+the storageos operator and install storageos. Also includes scripts to
+uninstallation everything it installs.
+
+
+## community-changes.yaml, rhel-changes.yaml, package-changes.yaml
+
+These files are update scripts for the yaml files used by yq yaml processor.

deploy/olm/community-changes.yaml

@@ -0,0 +1,5 @@
+metadata.name: storageosoperator.v1.1.0
+metadata.namespace: placeholder
+metadata.annotations.containerImage: storageos/cluster-operator:1.1.0
+spec.version: 1.1.0
+spec.install.spec.deployments[0].spec.template.spec.containers[0].image: storageos/cluster-operator:1.1.0

deploy/olm/csv-rhel/README.md

@@ -10,9 +10,16 @@ change, a new CSV with incremented version must be created.
 ## Release Instructions
 
 To create a new release:
-1. Update `metadata.annotations.containerImage` and deployment image with the
-new operator container image and `spec.version` to the new release version
-number in `storageos.clusterserviceversion.yaml`.
+1. Update `metadata.name` with new version name,
+`metadata.annotations.containerImage` and
+`spec.install.spec.deployments[0].spec.template.spec.containers[0].image` with
+the new operator container image, and `spec.version` to the new release version
+number in `deploy/olm/rhel-changes.yaml`. Run `make metadata-update` to generate
+`storageos.clusterserviceversion.yaml`.
+Any other change in the CSV file must be made in
+`deploy/storageos-operators.configmap.yaml` and regenerate all the metadata
+files.
+
 2. Run `make metadata-zip` from the root of the project to generate a metadata
 zip file at `/build/_output/storageos-olm-metadata.zip`. This file can be
 directly uploaded to the rhel operator metadata scanner for a new release.
@@ -23,4 +30,4 @@ the repo to keep a record of the releases.
 ## Testing
 
 Run `make metadata-bundle-lint` from the root of the project to create a
-metadata bundle and lint it before submitting a new release.
+metadata bundle at `build/_output/` and lint it before submitting a new release.

deploy/olm/csv-rhel/storageos.clusterserviceversion.yaml

@@ -5,11 +5,11 @@ metadata:
   namespace: placeholder
   annotations:
     capabilities: Full Lifecycle
-    categories: "Storage"
+    categories: Storage
     description: Cloud-native, persistent storage for containers.
     containerImage: registry.connect.redhat.com/storageos/cluster-operator:1.1.0
     repository: https://github.com/storageos/cluster-operator
-    createdAt: 2019-04-17T08:00:00Z
+    createdAt: "2019-04-17T08:00:00Z"
     support: StorageOS, Inc
     certified: "true"
     alm-examples: |-
@@ -54,7 +54,6 @@ metadata:
           }
         }
       ]
-
 spec:
   displayName: StorageOS
   description: |
@@ -151,36 +150,32 @@ spec:
     on any platform while maintaining full control of business requirements
     around availability, data mobility, performance, security, data residency
     compliance and business continuity.
-
-  keywords: ['storageos', 'storage', 'persistent storage', 'open source']
-
+  keywords:
+  - storageos
+  - storage
+  - persistent storage
+  - open source
   version: 1.1.0
-  minKubeVersion: "1.10.0"
+  minKubeVersion: 1.10.0
   maturity: stable
   maintainers:
   - name: StorageOS, Inc
     email: [email protected]
-
   provider:
     name: StorageOS, Inc
-
   labels:
     operated-by: storageosoperator
-
   selector:
     matchLabels:
       operated-by: storageosoperator
-
   links:
   - name: Documentation
     url: https://docs.storageos.com/
   - name: StorageOS Operator Source Code
     url: https://github.com/storageos/cluster-operator
-
   icon:
   - base64data: PD94bWwgdmVyc2lvbj0iMS4wIiBlbmNvZGluZz0idXRmLTgiPz4NCjwhLS0gR2VuZXJhdG9yOiBBZG9iZSBJbGx1c3RyYXRvciAxOS4wLjAsIFNWRyBFeHBvcnQgUGx1Zy1JbiAuIFNWRyBWZXJzaW9uOiA2LjAwIEJ1aWxkIDApICAtLT4NCjxzdmcgdmVyc2lvbj0iMS4xIiBpZD0iTGF5ZXJfMSIgeG1sbnM9Imh0dHA6Ly93d3cudzMub3JnLzIwMDAvc3ZnIiB4bWxuczp4bGluaz0iaHR0cDovL3d3dy53My5vcmcvMTk5OS94bGluayIgeD0iMHB4IiB5PSIwcHgiDQoJIHZpZXdCb3g9Ii05MCAyODQuNyA0MzAgNDMwIiBlbmFibGUtYmFja2dyb3VuZD0ibmV3IC05MCAyODQuNyA0MzAgNDMwIiB4bWw6c3BhY2U9InByZXNlcnZlIj4NCjxnPg0KCTxwYXRoIGZpbGw9IiM0RjUyNjMiIGQ9Ik0yNzguNSw0MjAuOWgtNzQuMWwtNS42LTkuN2wxMy42LTIzLjZsLTE4LjEtMzEuM0gxODBsMjcuNS00Ny43YzMtNS4xLDEuMi0xMS43LTMuOS0xNC42DQoJCWMtNS4xLTMtMTEuNy0xLjItMTQuNiwzLjlsLTM0LDU4LjhIOTMuMmwtMzQtNTguOGMtMi45LTUuMS05LjUtNi45LTE0LjYtMy45cy02LjksOS41LTMuOSwxNC42bDI3LjUsNDcuN0g1My45bC0xOC4xLDMxLjMNCgkJbDEzLjYsMjMuNmwtNS42LDkuOGgtNzQuMmwtNDMuMyw3NC45bDQzLjMsNzQuOWg3NC4xbDM0LDU4LjlMNzksNjI5bC0xLjIsMC44bDQ2LjQsNzcuMmw0Ni4yLTc3LjNsMCwwbDM0LTU4LjloNzQuMWw0My4zLTc0LjkNCgkJTDI3OC41LDQyMC45eiBNNTUuNSw0MDAuNWwtNy40LTEyLjhMNjAsMzY3aDE0LjNsMC4zLDAuNUw1NS41LDQwMC41eiBNMTczLjQsMzY3LjVsMC4zLTAuNUgxODhsMTEuOSwyMC42bC03LjQsMTIuOUwxNzMuNCwzNjcuNXoiDQoJCS8+DQoJPHBvbHlnb24gZmlsbD0iI0ZGRkZGRiIgcG9pbnRzPSI0My43LDQ0Mi4zIDQyLjgsNDQyLjMgLTE4LjEsNDQyLjMgLTQ4LjksNDk1LjggLTE4LjEsNTQ5LjMgNDMsNTQ5LjMgNDMuNyw1NDkuMyA3NC42LDQ5NS45IAkiLz4NCgk8cG9seWdvbiBmaWxsPSIjRkZGRkZGIiBwb2ludHM9IjI2Ni4xLDQ0Mi4zIDIwNC4zLDQ0Mi4zIDIwNC4zLDQ0Mi41IDE3My40LDQ5NS44IDIwNC4zLDU0OS4zIDIwNC40LDU0OS4zIDI2Ni4xLDU0OS4zIDI5Nyw0OTUuOCAJDQoJCSIvPg0KCTxwb2x5Z29uIGZpbGw9IiM2MUMyMDIiIHBvaW50cz0iMTU0LjksMzc4LjIgOTMuMSwzNzguMiA5My4xLDM3OC4zIDYyLjMsNDMxLjcgOTMuMSw0ODUuMiA5My4yLDQ4NS4yIDE1NC45LDQ4NS4yIDE4NS44LDQzMS43IAkiLz4NCgk8cG9seWdvbiBmaWxsPSIjNjFDMjAyIiBwb2ludHM9IjE1Mi4xLDYxOC40IDE1Mi4xLDYxOC40IDEyNCw2NjUuMiA5Ni4yLDYxOC45IDk2LjIsNjE4LjkgNjIuMiw1NjAuMSA4OC4zLDUxNC45IDkzLjEsNTA2LjYgDQoJCTE1NC45LDUwNi42IDE4NS44LDU2MC4xIAkiLz4NCjwvZz4NCjxyZWN0IHg9Ii0xMDUiIHk9IjI3MC43IiBmaWxsPSJub25lIiB3aWR0aD0iNDU4IiBoZWlnaHQ9IjQ1OCIvPg0KPC9zdmc+DQo=
     mediatype: image/svg+xml
-
   installModes:
   - type: OwnNamespace
     supported: true
@@ -190,7 +185,6 @@ spec:
     supported: false
   - type: AllNamespaces
     supported: true
-
   install:
     strategy: deployment
     spec:
@@ -204,14 +198,14 @@ spec:
           - storageosupgrades
           - jobs
           verbs:
-          - "*"
+          - '*'
         - apiGroups:
           - apps
           resources:
           - statefulsets
           - daemonsets
           verbs:
-          - "*"
+          - '*'
         - apiGroups:
           - ""
           resources:
@@ -324,7 +318,7 @@ spec:
                     fieldRef:
                       fieldPath: metadata.annotations['olm.targetNamespaces']
                 - name: OPERATOR_NAME
-                  value: "cluster-operator"
+                  value: cluster-operator
                 ports:
                 - containerPort: 8080
                   name: metrics
@@ -334,34 +328,34 @@ spec:
       version: v1
       kind: StorageOSCluster
       displayName: StorageOS Cluster
-      description: StorageOS Cluster installs StorageOS in the cluster. It
-        contains all the configuration for setting up a StorageOS cluster and
-        also shows the status of the running StorageOS cluster.
+      description: StorageOS Cluster installs StorageOS in the cluster. It contains
+        all the configuration for setting up a StorageOS cluster and also shows the
+        status of the running StorageOS cluster.
       specDescriptors:
       - description: Defines the various container images used in the cluster.
         displayName: Images
         path: images
-      - description: The namespace to install the StorageOS cluster into.
-          `kube-system` is recommended so that StorageOS does not get evicted if
-          a node becomes over-allocated.
+      - description: The namespace to install the StorageOS cluster into. `kube-system`
+          is recommended so that StorageOS does not get evicted if a node becomes
+          over-allocated.
         displayName: Namespace
         path: namespace
       - description: The name of the secret object that stores the api credentials.
         displayName: Cluster Secret
         path: secretRefName
-      - description: The name of the namespace where the secret object that
-          stores the api credentials exists.
+      - description: The name of the namespace where the secret object that stores
+          the api credentials exists.
         displayName: Cluster Secret Namespace
         path: secretRefNamespace
-      - description: The join token is used for cluster discovery.  When used
-          with the Operator, the token will be a comma-separated list of all
-          cluster member IP addresses.  The node that owns the first IP address
-          listed will be responsible for bootsrapping the cluster.
+      - description: The join token is used for cluster discovery.  When used with
+          the Operator, the token will be a comma-separated list of all cluster member
+          IP addresses.  The node that owns the first IP address listed will be responsible
+          for bootsrapping the cluster.
         displayName: Cluster members
         path: join
-      - description: KV store configuration to use. Defaults to embedded. `etcd`
-          is recommended for production deployments with the address set to an
-          external etcd instance.
+      - description: KV store configuration to use. Defaults to embedded. `etcd` is
+          recommended for production deployments with the address set to an external
+          etcd instance.
         displayName: KV Store
         path: kvBackend
       - description: Describes the Container Storage Interface (CSI) configuration.
@@ -370,52 +364,50 @@ spec:
       - description: The cluster Service configuration.
         displayName: Service configuration
         path: service
-      - description: The shared directory where storage devices should be
-          created.  This directory must be available to both the StorageOS Node
-          container and the kubelet, and must have mount propagation enabled.
-          When kubelet is running in a container,
-          `/var/lib/kubelet/plugins/kubernetes.io~storageos` should normally be
-          set, otherwise leave empty.
+      - description: The shared directory where storage devices should be created.  This
+          directory must be available to both the StorageOS Node container and the
+          kubelet, and must have mount propagation enabled. When kubelet is running
+          in a container, `/var/lib/kubelet/plugins/kubernetes.io~storageos` should
+          normally be set, otherwise leave empty.
         displayName: Device directory
         path: sharedDir
-      - description: Describes the ingress configuration to be configured for
-          the cluster.
+      - description: Describes the ingress configuration to be configured for the
+          cluster.
         displayName: Ingress configuration
         path: ingress
-      - description: The name of the secret object that contains the etcd TLS
-          certificates.
+      - description: The name of the secret object that contains the etcd TLS certificates.
         displayName: etcd TLS Secret Name
         path: tlsEtcdSecretRefName
-      - description: The namespace of the secret object that contains the etcd
-          TLS certificates.
+      - description: The namespace of the secret object that contains the etcd TLS
+          certificates.
         displayName: etcd TLS Secret Namespace
         path: tlsEtcdSecretRefNamespace
-      - description: Node selector terms can be set to control the placement of
-          StorageOS pods using node affiinity.
+      - description: Node selector terms can be set to control the placement of StorageOS
+          pods using node affiinity.
         displayName: Node Selectors
         path: nodeSelectorTerms
-      - description: Tolerations can be set to control the placement of
-          StorageOS pods.
+      - description: Tolerations can be set to control the placement of StorageOS
+          pods.
         displayName: Tolerations
         path: tolerations
-      - description: Name of the Kubernetes distribution in use, e.g.
-          `openshift`.  This will be included in the product telemetry (if
-          enabled), to help focus development efforts.
+      - description: Name of the Kubernetes distribution in use, e.g. `openshift`.  This
+          will be included in the product telemetry (if enabled), to help focus development
+          efforts.
         displayName: Kubernetes Distribution Name
         path: k8sDistro
-      - description: To disable anonymous usage reporting across the cluster,
-          set to true. Defaults to false. To help improve the product, data such
-          as API usage and StorageOS configuration information is collected.
+      - description: To disable anonymous usage reporting across the cluster, set
+          to true. Defaults to false. To help improve the product, data such as API
+          usage and StorageOS configuration information is collected.
         displayName: Disable Telemetry
         path: disableTelemetry
-      - description: When Pod Fencing is disabled, StorageOS will not perform
-          any interaction with Kubernetes when it detects that a node has gone
-          offline. Additionally, the Kubernetes permissions required for Fencing
-          will not be added to the StorageOS role.
+      - description: When Pod Fencing is disabled, StorageOS will not perform any
+          interaction with Kubernetes when it detects that a node has gone offline.
+          Additionally, the Kubernetes permissions required for Fencing will not be
+          added to the StorageOS role.
         displayName: Disable Fencing
         path: disableFencing
-      - description: When enabled, the Operator will not perform any actions on
-          the cluster.
+      - description: When enabled, the Operaaor will not perform any actions on the
+          cluster.
         displayName: Pause Operator
         path: pause
       - description: Enables debug logging when set to true.
@@ -442,9 +434,9 @@ spec:
       version: v1
       kind: Job
       displayName: StorageOS Job
-      description: StorageOS Job creates special pods that run on all the node
-        and perform an administrative task. This could be used for cluster
-        maintenance tasks.
+      description: StorageOS Job creates special pods that run on all the node and
+        perform an administrative task. This could be used for cluster maintenance
+        tasks.
       specDescriptors:
       - description: The container image to run as the job.
         displayName: Image
@@ -458,15 +450,15 @@ spec:
       - description: The path on the host that is mounted into the job container.
         displayName: Source path
         path: hostPath
-      - description: The job is marked as completed when the completion word is
-          found in the pod logs.
+      - description: The job is marked as completed when the completion word is found
+          in the pod logs.
         displayName: Source Path
         path: completionWord
       - description: A label selector can be set to identify Pods created by the job.
         displayName: Label Selector
         path: labelSelector
-      - description: Node selector terms can be set to control the placement of
-          job pods using node affiinity.
+      - description: Node selector terms can be set to control the placement of job
+          pods using node affiinity.
         displayName: Node Selectors
         path: nodeSelectorTerms
       - description: Tolerations can be set to control the placement of job pods.
@@ -480,11 +472,10 @@ spec:
       version: v1
       kind: StorageOSUpgrade
       displayName: StorageOS Upgrade
-      description: StorageOS Upgrade automatically upgrades an existing
-        StorageOS cluster as per the upgrade configuration.
+      description: StorageOS Upgrade automatically upgrades an existing StorageOS
+        cluster as per the upgrade configuration.
       specDescriptors:
-      - description: The StorageOS Node image to upgrade to.
-          e.g. `registry.connect.redhat.com/storageos/node:latest`
+      - description: The StorageOS Node image to upgrade to. e.g. `registry.connect.redhat.com/storageos/node:latest`
         displayName: New Image
         path: newImage
       statusDescriptors:

deploy/olm/package-changes.yaml

@@ -0,0 +1 @@
+channels[0].currentCSV: storageosoperator.v1.1.0