diff --git a/ci/scripts/run-k8s-test-network-basic.sh b/ci/scripts/run-k8s-test-network-basic.sh index 6ce21f47..35e9e93d 100755 --- a/ci/scripts/run-k8s-test-network-basic.sh +++ b/ci/scripts/run-k8s-test-network-basic.sh @@ -28,12 +28,12 @@ export GATEWAY_CLIENT_APPLICATION_PATH=${GATEWAY_CLIENT_APPLICATION_PATH:-../ass export CHANNEL_NAME=${TEST_NETWORK_CHANNEL_NAME:-mychannel} export CHAINCODE_NAME=${TEST_NETWORK_CHAINCODE_NAME:-asset-transfer-basic} export MSP_ID=${MSP_ID:-Org1MSP} -export CRYPTO_PATH=${CRYPTO_PATH:-../../test-network-k8s/build/organizations/peerOrganizations/org1.example.com} -export KEY_DIRECTORY_PATH=${KEY_DIRECTORY_PATH:-../../test-network-k8s/build/organizations/peerOrganizations/org1.example.com/users/Admin@org1.example.com/msp/keystore} -export CERT_PATH=${CERT_PATH:-../../test-network-k8s/build/organizations/peerOrganizations/org1.example.com/users/Admin@org1.example.com/msp/signcerts/cert.pem} -export TLS_CERT_PATH=${TLS_CERT_PATH:-../../test-network-k8s/build/organizations/peerOrganizations/org1.example.com/msp/tlscacerts/org1-tls-ca.pem} -export PEER_ENDPOINT=${PEER_ENDPOINT:-localhost:7051} -export PEER_HOST_ALIAS=${PEER_HOST_ALIAS:-org1-peer1} +export CRYPTO_PATH=${CRYPTO_PATH:-../../test-network-k8s/build/channel-msp/peerOrganizations/org1} +export KEY_DIRECTORY_PATH=${KEY_DIRECTORY_PATH:-../../test-network-k8s/build/enrollments/org1/users/org1admin/msp/keystore} +export CERT_PATH=${CERT_PATH:-../../test-network-k8s/build/enrollments/org1/users/org1admin/msp/signcerts/cert.pem} +export TLS_CERT_PATH=${TLS_CERT_PATH:-../../test-network-k8s/build/channel-msp/peerOrganizations/org1/msp/tlscacerts/tlsca-signcert.pem} +export PEER_ENDPOINT=${PEER_ENDPOINT:-org1-peer1.vcap.me:443} +export PEER_HOST_ALIAS=${PEER_HOST_ALIAS:-org1-peer1.vcap.me} function print() { GREEN='\033[0;32m' @@ -80,29 +80,13 @@ function createNetwork() { ./network up ./network channel create - print "Opening gateway port-forward to 'localhost:7051'" - kubectl -n test-network port-forward svc/org1-peer1 7051:7051 & - print "Deploying chaincode" ./network chaincode deploy - - print "Extracting certificates" - kubectl \ - -n test-network \ - exec deploy/org1-peer1 \ - -c main \ - -- tar zcf - -C /var/hyperledger/fabric organizations/peerOrganizations/org1.example.com \ - | tar zxvf - -C ../test-network-k8s/build/ } function stopNetwork() { - pkill -f "port-forward" - print "Stopping network" ./network down - - print "Cleaning client certificates" - rm -rf ../test-network-k8s/build/ } # Set up the suite with a KIND cluster @@ -122,8 +106,7 @@ print "Running rest-easy test" ( ./network rest-easy \ && sleep 5 \ && export SAMPLE_APIKEY='97834158-3224-4CE7-95F9-A148C886653E' \ - && curl -s --header "X-Api-Key: ${SAMPLE_APIKEY}" "http://localhost/api/assets/asset1" | jq \ - && curl -s --insecure --header "X-Api-Key: ${SAMPLE_APIKEY}" "https://localhost/api/assets/asset1" | jq ) + && curl -s --header "X-Api-Key: ${SAMPLE_APIKEY}" "http://fabric-rest-sample.vcap.me/api/assets/asset1" | jq ) print "OK" stopNetwork diff --git a/test-network-k8s/.gitignore b/test-network-k8s/.gitignore index b4afee58..c1f650e1 100644 --- a/test-network-k8s/.gitignore +++ b/test-network-k8s/.gitignore @@ -3,3 +3,4 @@ network.log network-debug.log build/ .env +bin/ diff --git a/test-network-k8s/config/org0/configtx.yaml b/test-network-k8s/config/org0/configtx.yaml index 337c83c2..2981b261 100644 --- a/test-network-k8s/config/org0/configtx.yaml +++ b/test-network-k8s/config/org0/configtx.yaml @@ -25,7 +25,7 @@ Organizations: ID: OrdererMSP # MSPDir is the filesystem path which contains the MSP configuration - MSPDir: /var/hyperledger/fabric/organizations/ordererOrganizations/org0.example.com/msp + MSPDir: ../../build/channel-msp/ordererOrganizations/org0/msp # Policies defines the set of policies at this level of the config tree # For organization policies, their canonical path is usually @@ -54,7 +54,7 @@ Organizations: # ID to load the MSP definition as ID: Org1MSP - MSPDir: /var/hyperledger/fabric/organizations/peerOrganizations/org1.example.com/msp + MSPDir: ../../build/channel-msp/peerOrganizations/org1/msp # Policies defines the set of policies at this level of the config tree # For organization policies, their canonical path is usually @@ -89,7 +89,7 @@ Organizations: # ID to load the MSP definition as ID: Org2MSP - MSPDir: /var/hyperledger/fabric/organizations/peerOrganizations/org2.example.com/msp + MSPDir: ../../build/channel-msp/peerOrganizations/org2/msp # Policies defines the set of policies at this level of the config tree # For organization policies, their canonical path is usually @@ -224,16 +224,16 @@ Orderer: &OrdererDefaults Consenters: - Host: org0-orderer1 Port: 6050 - ClientTLSCert: /var/hyperledger/fabric/organizations/ordererOrganizations/org0.example.com/orderers/org0-orderer1.org0.example.com/tls/signcerts/cert.pem - ServerTLSCert: /var/hyperledger/fabric/organizations/ordererOrganizations/org0.example.com/orderers/org0-orderer1.org0.example.com/tls/signcerts/cert.pem + ClientTLSCert: ../../build/channel-msp/ordererOrganizations/org0/orderers/org0-orderer1/tls/signcerts/tls-cert.pem + ServerTLSCert: ../../build/channel-msp/ordererOrganizations/org0/orderers/org0-orderer1/tls/signcerts/tls-cert.pem - Host: org0-orderer2 Port: 6050 - ClientTLSCert: /var/hyperledger/fabric/organizations/ordererOrganizations/org0.example.com/orderers/org0-orderer2.org0.example.com/tls/signcerts/cert.pem - ServerTLSCert: /var/hyperledger/fabric/organizations/ordererOrganizations/org0.example.com/orderers/org0-orderer2.org0.example.com/tls/signcerts/cert.pem + ClientTLSCert: ../../build/channel-msp/ordererOrganizations/org0/orderers/org0-orderer2/tls/signcerts/tls-cert.pem + ServerTLSCert: ../../build/channel-msp/ordererOrganizations/org0/orderers/org0-orderer2/tls/signcerts/tls-cert.pem - Host: org0-orderer3 Port: 6050 - ClientTLSCert: /var/hyperledger/fabric/organizations/ordererOrganizations/org0.example.com/orderers/org0-orderer3.org0.example.com/tls/signcerts/cert.pem - ServerTLSCert: /var/hyperledger/fabric/organizations/ordererOrganizations/org0.example.com/orderers/org0-orderer3.org0.example.com/tls/signcerts/cert.pem + ClientTLSCert: ../../build/channel-msp/ordererOrganizations/org0/orderers/org0-orderer3/tls/signcerts/tls-cert.pem + ServerTLSCert: ../../build/channel-msp/ordererOrganizations/org0/orderers/org0-orderer3/tls/signcerts/tls-cert.pem # Options to be specified for all the etcd/raft nodes. The values here diff --git a/test-network-k8s/kube/fabric-rest-sample.yaml b/test-network-k8s/kube/fabric-rest-sample.yaml index b8803101..32be0f0a 100644 --- a/test-network-k8s/kube/fabric-rest-sample.yaml +++ b/test-network-k8s/kube/fabric-rest-sample.yaml @@ -236,23 +236,26 @@ spec: selector: app: fabric-rest-sample - --- apiVersion: networking.k8s.io/v1 kind: Ingress metadata: + annotations: + nginx.ingress.kubernetes.io/proxy-connect-timeout: 60s + labels: + app: fabric-rest-sample name: fabric-rest-sample -# annotations: -# nginx.ingress.kubernetes.io/rewrite-target: /$1 spec: + ingressClassName: nginx rules: - - http: + - host: fabric-rest-sample.${DOMAIN} + http: paths: -# - path: "/fabric-rest-sample/(.*)" - - path: "/" - pathType: Prefix - backend: + - backend: service: name: fabric-rest-sample port: - number: 3000 + name: http + path: / + pathType: ImplementationSpecific + diff --git a/test-network-k8s/kube/org0/org0-admin-cli.yaml b/test-network-k8s/kube/org0/org0-admin-cli.yaml deleted file mode 100644 index 85c81343..00000000 --- a/test-network-k8s/kube/org0/org0-admin-cli.yaml +++ /dev/null @@ -1,61 +0,0 @@ -# -# Copyright IBM Corp. All Rights Reserved. -# -# SPDX-License-Identifier: Apache-2.0 -# ---- -apiVersion: apps/v1 -kind: Deployment -metadata: - name: org0-admin-cli -spec: - replicas: 1 - selector: - matchLabels: - app: org0-admin-cli - template: - metadata: - labels: - app: org0-admin-cli - spec: - containers: - - name: main - image: {{FABRIC_CONTAINER_REGISTRY}}/fabric-tools:{{FABRIC_VERSION}} - imagePullPolicy: IfNotPresent - env: - - name: FABRIC_CFG_PATH - value: /var/hyperledger/fabric/config - args: - - sleep - - "2147483647" - workingDir: /root - volumeMounts: - - name: fabric-volume - mountPath: /var/hyperledger - - name: fabric-config - mountPath: /var/hyperledger/fabric/config - - # This init container will unfurl all of the MSP archives listed in the msp-config config map. - initContainers: - - name: msp-unfurl - image: busybox - command: - - sh - - -c - - "for msp in $(ls /msp/msp-*.tgz); do echo $msp && tar zxvf $msp -C /var/hyperledger/fabric; done" - volumeMounts: - - name: msp-config - mountPath: /msp - - name: fabric-volume - mountPath: /var/hyperledger - - volumes: - - name: fabric-volume - persistentVolumeClaim: - claimName: fabric-org0 - - name: fabric-config - configMap: - name: org0-config - - name: msp-config - configMap: - name: msp-config diff --git a/test-network-k8s/kube/org0/org0-ca.yaml b/test-network-k8s/kube/org0/org0-ca.yaml index d79a8737..b4e4ac2a 100644 --- a/test-network-k8s/kube/org0/org0-ca.yaml +++ b/test-network-k8s/kube/org0/org0-ca.yaml @@ -17,6 +17,7 @@ spec: - localhost - org0-ca - org0-ca.test-network.svc.cluster.local + - org0-ca.${DOMAIN} ipAddresses: - 127.0.0.1 secretName: org0-ca-tls-cert @@ -40,7 +41,7 @@ spec: spec: containers: - name: main - image: {{FABRIC_CONTAINER_REGISTRY}}/fabric-ca:{{FABRIC_CA_VERSION}} + image: ${FABRIC_CONTAINER_REGISTRY}/fabric-ca:${FABRIC_CA_VERSION} imagePullPolicy: IfNotPresent env: - name: FABRIC_CA_SERVER_CA_NAME @@ -89,8 +90,35 @@ metadata: name: org0-ca spec: ports: - - name: tls + - name: https port: 443 protocol: TCP selector: - app: org0-ca \ No newline at end of file + app: org0-ca + +--- +apiVersion: networking.k8s.io/v1 +kind: Ingress +metadata: + annotations: + nginx.ingress.kubernetes.io/proxy-connect-timeout: 60s + nginx.ingress.kubernetes.io/ssl-passthrough: "true" + labels: + app: org0-ca + name: org0-ca +spec: + ingressClassName: nginx + rules: + - host: org0-ca.${DOMAIN} + http: + paths: + - backend: + service: + name: org0-ca + port: + name: https + path: / + pathType: ImplementationSpecific + tls: + - hosts: + - org0-ca.${DOMAIN} diff --git a/test-network-k8s/kube/org0/org0-orderer1.yaml b/test-network-k8s/kube/org0/org0-orderer1.yaml index fdef3938..f8bf9c36 100644 --- a/test-network-k8s/kube/org0/org0-orderer1.yaml +++ b/test-network-k8s/kube/org0/org0-orderer1.yaml @@ -9,7 +9,7 @@ apiVersion: cert-manager.io/v1 kind: Certificate metadata: name: org0-orderer1-tls-cert - namespace: test-network + namespace: ${NS} spec: isCA: false privateKey: @@ -18,7 +18,9 @@ spec: dnsNames: - localhost - org0-orderer1 - - org0-orderer1.test-network.svc.cluster.local + - org0-orderer1.${NS}.svc.cluster.local + - org0-orderer1.${DOMAIN} + - org0-orderer1-admin.${DOMAIN} ipAddresses: - 127.0.0.1 secretName: org0-orderer1-tls-cert @@ -42,6 +44,12 @@ data: ORDERER_GENERAL_TLS_ROOTCAS: /var/hyperledger/fabric/config/tls/ca.crt ORDERER_GENERAL_TLS_PRIVATEKEY: /var/hyperledger/fabric/config/tls/tls.key ORDERER_GENERAL_BOOTSTRAPMETHOD: none + ORDERER_ADMIN_TLS_ENABLED: "true" + ORDERER_ADMIN_TLS_CERTIFICATE: /var/hyperledger/fabric/config/tls/tls.crt + ORDERER_ADMIN_TLS_ROOTCAS: /var/hyperledger/fabric/config/tls/ca.crt + ORDERER_ADMIN_TLS_PRIVATEKEY: /var/hyperledger/fabric/config/tls/tls.key + # Authenticate client connections with the org's ecert / admin user enrollments + ORDERER_ADMIN_TLS_CLIENTROOTCAS: "[/var/hyperledger/fabric/organizations/ordererOrganizations/org0.example.com/orderers/org0-orderer1.org0.example.com/msp/cacerts/org0-ca.pem]" ORDERER_FILELEDGER_LOCATION: /var/hyperledger/fabric/data/orderer1 ORDERER_CONSENSUS_WALDIR: /var/hyperledger/fabric/data/orderer1/etcdraft/wal ORDERER_CONSENSUS_SNAPDIR: /var/hyperledger/fabric/data/orderer1/etcdraft/wal @@ -65,7 +73,7 @@ spec: spec: containers: - name: main - image: {{FABRIC_CONTAINER_REGISTRY}}/fabric-orderer:{{FABRIC_VERSION}} + image: ${FABRIC_CONTAINER_REGISTRY}/fabric-orderer:${FABRIC_VERSION} imagePullPolicy: IfNotPresent envFrom: - configMapRef: @@ -109,4 +117,43 @@ spec: port: 9443 protocol: TCP selector: - app: org0-orderer1 \ No newline at end of file + app: org0-orderer1 + +--- +apiVersion: networking.k8s.io/v1 +kind: Ingress +metadata: + annotations: + nginx.ingress.kubernetes.io/proxy-connect-timeout: 60s + nginx.ingress.kubernetes.io/ssl-passthrough: "true" + labels: + app: org0-orderer1 + name: org0-orderer1 +spec: + ingressClassName: nginx + rules: + - host: org0-orderer1.${DOMAIN} + http: + paths: + - backend: + service: + name: org0-orderer1 + port: + name: general + path: / + pathType: ImplementationSpecific + - host: org0-orderer1-admin.${DOMAIN} + http: + paths: + - backend: + service: + name: org0-orderer1 + port: + name: admin + path: / + pathType: ImplementationSpecific + tls: + - hosts: + - org0-orderer1.${DOMAIN} + - hosts: + - org0-orderer1-admin.${DOMAIN} diff --git a/test-network-k8s/kube/org0/org0-orderer2.yaml b/test-network-k8s/kube/org0/org0-orderer2.yaml index 23e0cb61..fa96f90e 100644 --- a/test-network-k8s/kube/org0/org0-orderer2.yaml +++ b/test-network-k8s/kube/org0/org0-orderer2.yaml @@ -3,12 +3,13 @@ # # SPDX-License-Identifier: Apache-2.0 # + --- apiVersion: cert-manager.io/v1 kind: Certificate metadata: name: org0-orderer2-tls-cert - namespace: test-network + namespace: ${NS} spec: isCA: false privateKey: @@ -17,7 +18,9 @@ spec: dnsNames: - localhost - org0-orderer2 - - org0-orderer2.test-network.svc.cluster.local + - org0-orderer2.${NS}.svc.cluster.local + - org0-orderer2.${DOMAIN} + - org0-orderer2-admin.${DOMAIN} ipAddresses: - 127.0.0.1 secretName: org0-orderer2-tls-cert @@ -41,6 +44,12 @@ data: ORDERER_GENERAL_TLS_ROOTCAS: /var/hyperledger/fabric/config/tls/ca.crt ORDERER_GENERAL_TLS_PRIVATEKEY: /var/hyperledger/fabric/config/tls/tls.key ORDERER_GENERAL_BOOTSTRAPMETHOD: none + ORDERER_ADMIN_TLS_ENABLED: "true" + ORDERER_ADMIN_TLS_CERTIFICATE: /var/hyperledger/fabric/config/tls/tls.crt + ORDERER_ADMIN_TLS_ROOTCAS: /var/hyperledger/fabric/config/tls/ca.crt + ORDERER_ADMIN_TLS_PRIVATEKEY: /var/hyperledger/fabric/config/tls/tls.key + # Authenticate client connections with the org's ecert / admin user enrollments + ORDERER_ADMIN_TLS_CLIENTROOTCAS: "[/var/hyperledger/fabric/organizations/ordererOrganizations/org0.example.com/orderers/org0-orderer2.org0.example.com/msp/cacerts/org0-ca.pem]" ORDERER_FILELEDGER_LOCATION: /var/hyperledger/fabric/data/orderer2 ORDERER_CONSENSUS_WALDIR: /var/hyperledger/fabric/data/orderer2/etcdraft/wal ORDERER_CONSENSUS_SNAPDIR: /var/hyperledger/fabric/data/orderer2/etcdraft/wal @@ -64,7 +73,7 @@ spec: spec: containers: - name: main - image: {{FABRIC_CONTAINER_REGISTRY}}/fabric-orderer:{{FABRIC_VERSION}} + image: ${FABRIC_CONTAINER_REGISTRY}/fabric-orderer:${FABRIC_VERSION} imagePullPolicy: IfNotPresent envFrom: - configMapRef: @@ -108,4 +117,43 @@ spec: port: 9443 protocol: TCP selector: - app: org0-orderer2 \ No newline at end of file + app: org0-orderer2 + +--- +apiVersion: networking.k8s.io/v1 +kind: Ingress +metadata: + annotations: + nginx.ingress.kubernetes.io/proxy-connect-timeout: 60s + nginx.ingress.kubernetes.io/ssl-passthrough: "true" + labels: + app: org0-orderer2 + name: org0-orderer2 +spec: + ingressClassName: nginx + rules: + - host: org0-orderer2.${DOMAIN} + http: + paths: + - backend: + service: + name: org0-orderer2 + port: + name: general + path: / + pathType: ImplementationSpecific + - host: org0-orderer2-admin.${DOMAIN} + http: + paths: + - backend: + service: + name: org0-orderer2 + port: + name: admin + path: / + pathType: ImplementationSpecific + tls: + - hosts: + - org0-orderer2.${DOMAIN} + - hosts: + - org0-orderer2-admin.${DOMAIN} diff --git a/test-network-k8s/kube/org0/org0-orderer3.yaml b/test-network-k8s/kube/org0/org0-orderer3.yaml index 40841146..3391f6a8 100644 --- a/test-network-k8s/kube/org0/org0-orderer3.yaml +++ b/test-network-k8s/kube/org0/org0-orderer3.yaml @@ -3,12 +3,13 @@ # # SPDX-License-Identifier: Apache-2.0 # + --- apiVersion: cert-manager.io/v1 kind: Certificate metadata: name: org0-orderer3-tls-cert - namespace: test-network + namespace: ${NS} spec: isCA: false privateKey: @@ -17,13 +18,15 @@ spec: dnsNames: - localhost - org0-orderer3 - - org0-orderer3.test-network.svc.cluster.local + - org0-orderer3.${NS}.svc.cluster.local + - org0-orderer3.${DOMAIN} + - org0-orderer3-admin.${DOMAIN} ipAddresses: - 127.0.0.1 secretName: org0-orderer3-tls-cert issuerRef: name: org0-tls-cert-issuer - + --- apiVersion: v1 kind: ConfigMap @@ -41,6 +44,12 @@ data: ORDERER_GENERAL_TLS_ROOTCAS: /var/hyperledger/fabric/config/tls/ca.crt ORDERER_GENERAL_TLS_PRIVATEKEY: /var/hyperledger/fabric/config/tls/tls.key ORDERER_GENERAL_BOOTSTRAPMETHOD: none + ORDERER_ADMIN_TLS_ENABLED: "true" + ORDERER_ADMIN_TLS_CERTIFICATE: /var/hyperledger/fabric/config/tls/tls.crt + ORDERER_ADMIN_TLS_ROOTCAS: /var/hyperledger/fabric/config/tls/ca.crt + ORDERER_ADMIN_TLS_PRIVATEKEY: /var/hyperledger/fabric/config/tls/tls.key + # Authenticate client connections with the org's ecert / admin user enrollments + ORDERER_ADMIN_TLS_CLIENTROOTCAS: "[/var/hyperledger/fabric/organizations/ordererOrganizations/org0.example.com/orderers/org0-orderer3.org0.example.com/msp/cacerts/org0-ca.pem]" ORDERER_FILELEDGER_LOCATION: /var/hyperledger/fabric/data/orderer3 ORDERER_CONSENSUS_WALDIR: /var/hyperledger/fabric/data/orderer3/etcdraft/wal ORDERER_CONSENSUS_SNAPDIR: /var/hyperledger/fabric/data/orderer3/etcdraft/wal @@ -64,7 +73,7 @@ spec: spec: containers: - name: main - image: {{FABRIC_CONTAINER_REGISTRY}}/fabric-orderer:{{FABRIC_VERSION}} + image: ${FABRIC_CONTAINER_REGISTRY}/fabric-orderer:${FABRIC_VERSION} imagePullPolicy: IfNotPresent envFrom: - configMapRef: @@ -108,4 +117,43 @@ spec: port: 9443 protocol: TCP selector: - app: org0-orderer3 \ No newline at end of file + app: org0-orderer3 + +--- +apiVersion: networking.k8s.io/v1 +kind: Ingress +metadata: + annotations: + nginx.ingress.kubernetes.io/proxy-connect-timeout: 60s + nginx.ingress.kubernetes.io/ssl-passthrough: "true" + labels: + app: org0-orderer3 + name: org0-orderer3 +spec: + ingressClassName: nginx + rules: + - host: org0-orderer3.${DOMAIN} + http: + paths: + - backend: + service: + name: org0-orderer3 + port: + name: general + path: / + pathType: ImplementationSpecific + - host: org0-orderer3-admin.${DOMAIN} + http: + paths: + - backend: + service: + name: org0-orderer3 + port: + name: admin + path: / + pathType: ImplementationSpecific + tls: + - hosts: + - org0-orderer3.${DOMAIN} + - hosts: + - org0-orderer3-admin.${DOMAIN} diff --git a/test-network-k8s/kube/org1/org1-admin-cli.yaml b/test-network-k8s/kube/org1/org1-admin-cli.yaml deleted file mode 100644 index 8086e732..00000000 --- a/test-network-k8s/kube/org1/org1-admin-cli.yaml +++ /dev/null @@ -1,65 +0,0 @@ -# -# Copyright IBM Corp. All Rights Reserved. -# -# SPDX-License-Identifier: Apache-2.0 -# ---- -apiVersion: apps/v1 -kind: Deployment -metadata: - name: org1-admin-cli -spec: - replicas: 1 - selector: - matchLabels: - app: org1-admin-cli - template: - metadata: - labels: - app: org1-admin-cli - spec: - containers: - - name: main - image: {{FABRIC_CONTAINER_REGISTRY}}/fabric-tools:{{FABRIC_VERSION}} - imagePullPolicy: IfNotPresent - env: - - name: FABRIC_CFG_PATH - value: /var/hyperledger/fabric/config - - name: CORE_PEER_MSPCONFIGPATH - value: /var/hyperledger/fabric/organizations/peerOrganizations/org1.example.com/users/Admin@org1.example.com/msp - - name: CORE_PEER_TLS_ROOTCERT_FILE - value: /var/hyperledger/fabric/organizations/peerOrganizations/org1.example.com/msp/tlscacerts/org1-tls-ca.pem - args: - - sleep - - "2147483647" - workingDir: /root - volumeMounts: - - name: fabric-volume - mountPath: /var/hyperledger - - name: fabric-config - mountPath: /var/hyperledger/fabric/config - - # This init container will unfurl all of the MSP archives listed in the msp-config config map. - initContainers: - - name: msp-unfurl - image: busybox - command: - - sh - - -c - - "for msp in $(ls /msp/msp-*.tgz); do echo $msp && tar zxvf $msp -C /var/hyperledger/fabric; done" - volumeMounts: - - name: msp-config - mountPath: /msp - - name: fabric-volume - mountPath: /var/hyperledger - - volumes: - - name: fabric-volume - persistentVolumeClaim: - claimName: fabric-org1 - - name: fabric-config - configMap: - name: org1-config - - name: msp-config - configMap: - name: msp-config diff --git a/test-network-k8s/kube/org1/org1-ca.yaml b/test-network-k8s/kube/org1/org1-ca.yaml index e3985ecd..9428971f 100644 --- a/test-network-k8s/kube/org1/org1-ca.yaml +++ b/test-network-k8s/kube/org1/org1-ca.yaml @@ -17,6 +17,7 @@ spec: - localhost - org1-ca - org1-ca.test-network.svc.cluster.local + - org1-ca.${DOMAIN} ipAddresses: - 127.0.0.1 secretName: org1-ca-tls-cert @@ -40,7 +41,7 @@ spec: spec: containers: - name: main - image: {{FABRIC_CONTAINER_REGISTRY}}/fabric-ca:{{FABRIC_CA_VERSION}} + image: ${FABRIC_CONTAINER_REGISTRY}/fabric-ca:${FABRIC_CA_VERSION} imagePullPolicy: IfNotPresent env: - name: FABRIC_CA_SERVER_CA_NAME @@ -88,8 +89,36 @@ metadata: name: org1-ca spec: ports: - - name: tls + - name: https port: 443 protocol: TCP selector: - app: org1-ca \ No newline at end of file + app: org1-ca + +--- +apiVersion: networking.k8s.io/v1 +kind: Ingress +metadata: + annotations: + nginx.ingress.kubernetes.io/proxy-connect-timeout: 60s + nginx.ingress.kubernetes.io/ssl-passthrough: "true" + labels: + app: org1-ca + name: org1-ca +spec: + ingressClassName: nginx + rules: + - host: org1-ca.${DOMAIN} + http: + paths: + - backend: + service: + name: org1-ca + port: + name: https + path: / + pathType: ImplementationSpecific + tls: + - hosts: + - org1-ca.${DOMAIN} + \ No newline at end of file diff --git a/test-network-k8s/kube/org1/org1-peer1.yaml b/test-network-k8s/kube/org1/org1-peer1.yaml index 708c1db6..3f64b04a 100644 --- a/test-network-k8s/kube/org1/org1-peer1.yaml +++ b/test-network-k8s/kube/org1/org1-peer1.yaml @@ -17,8 +17,10 @@ spec: dnsNames: - localhost - org1-peer1 - - org1-peer-gateway-svc - org1-peer1.test-network.svc.cluster.local + - org1-peer1.${DOMAIN} + - org1-peer-gateway-svc + - org1-peer-gateway-svc.${DOMAIN} ipAddresses: - 127.0.0.1 secretName: org1-peer1-tls-cert @@ -74,7 +76,7 @@ spec: spec: containers: - name: main - image: {{FABRIC_CONTAINER_REGISTRY}}/fabric-peer:{{FABRIC_VERSION}} + image: ${FABRIC_CONTAINER_REGISTRY}/fabric-peer:${FABRIC_VERSION} imagePullPolicy: IfNotPresent envFrom: - configMapRef: @@ -118,7 +120,7 @@ metadata: name: org1-peer1 spec: ports: - - name: gossip + - name: grpc port: 7051 protocol: TCP - name: chaincode @@ -136,8 +138,46 @@ metadata: name: org1-peer-gateway-svc spec: ports: - - name: gossip + - name: grpc port: 7051 protocol: TCP selector: - org: org1 \ No newline at end of file + org: org1 +--- +apiVersion: networking.k8s.io/v1 +kind: Ingress +metadata: + annotations: + nginx.ingress.kubernetes.io/proxy-connect-timeout: 60s + nginx.ingress.kubernetes.io/ssl-passthrough: "true" + labels: + app: org1-peer1 + name: org1-peer1 +spec: + ingressClassName: nginx + rules: + - host: org1-peer1.${DOMAIN} + http: + paths: + - backend: + service: + name: org1-peer1 + port: + name: grpc + path: / + pathType: ImplementationSpecific + - host: org1-peer-gateway-svc.${DOMAIN} + http: + paths: + - backend: + service: + name: org1-peer1 + port: + name: grpc + path: / + pathType: ImplementationSpecific + tls: + - hosts: + - org1-peer1.${DOMAIN} + - hosts: + - org1-peer-gateway-svc.${DOMAIN} diff --git a/test-network-k8s/kube/org1/org1-peer2.yaml b/test-network-k8s/kube/org1/org1-peer2.yaml index 3bab4730..288536a8 100644 --- a/test-network-k8s/kube/org1/org1-peer2.yaml +++ b/test-network-k8s/kube/org1/org1-peer2.yaml @@ -19,6 +19,7 @@ spec: - org1-peer2 - org1-peer-gateway-svc - org1-peer2.test-network.svc.cluster.local + - org1-peer2.${DOMAIN} ipAddresses: - 127.0.0.1 secretName: org1-peer2-tls-cert @@ -75,7 +76,7 @@ spec: spec: containers: - name: main - image: {{FABRIC_CONTAINER_REGISTRY}}/fabric-peer:{{FABRIC_VERSION}} + image: ${FABRIC_CONTAINER_REGISTRY}/fabric-peer:${FABRIC_VERSION} imagePullPolicy: IfNotPresent envFrom: - configMapRef: @@ -119,7 +120,7 @@ metadata: name: org1-peer2 spec: ports: - - name: gossip + - name: grpc port: 7051 protocol: TCP - name: chaincode @@ -129,4 +130,30 @@ spec: port: 9443 protocol: TCP selector: - app: org1-peer2 \ No newline at end of file + app: org1-peer2 +--- +apiVersion: networking.k8s.io/v1 +kind: Ingress +metadata: + annotations: + nginx.ingress.kubernetes.io/proxy-connect-timeout: 60s + nginx.ingress.kubernetes.io/ssl-passthrough: "true" + labels: + app: org1-peer2 + name: org1-peer2 +spec: + ingressClassName: nginx + rules: + - host: org1-peer2.${DOMAIN} + http: + paths: + - backend: + service: + name: org1-peer2 + port: + name: grpc + path: / + pathType: ImplementationSpecific + tls: + - hosts: + - org1-peer2.${DOMAIN} \ No newline at end of file diff --git a/test-network-k8s/kube/org2/org2-admin-cli.yaml b/test-network-k8s/kube/org2/org2-admin-cli.yaml deleted file mode 100644 index 6839455a..00000000 --- a/test-network-k8s/kube/org2/org2-admin-cli.yaml +++ /dev/null @@ -1,65 +0,0 @@ -# -# Copyright IBM Corp. All Rights Reserved. -# -# SPDX-License-Identifier: Apache-2.0 -# ---- -apiVersion: apps/v1 -kind: Deployment -metadata: - name: org2-admin-cli -spec: - replicas: 1 - selector: - matchLabels: - app: org2-admin-cli - template: - metadata: - labels: - app: org2-admin-cli - spec: - containers: - - name: main - image: {{FABRIC_CONTAINER_REGISTRY}}/fabric-tools:{{FABRIC_VERSION}} - imagePullPolicy: IfNotPresent - env: - - name: FABRIC_CFG_PATH - value: /var/hyperledger/fabric/config - - name: CORE_PEER_MSPCONFIGPATH - value: /var/hyperledger/fabric/organizations/peerOrganizations/org2.example.com/users/Admin@org2.example.com/msp - - name: CORE_PEER_TLS_ROOTCERT_FILE - value: /var/hyperledger/fabric/organizations/peerOrganizations/org2.example.com/msp/tlscacerts/org2-tls-ca.pem - args: - - sleep - - "2147483647" - workingDir: /root - volumeMounts: - - name: fabric-volume - mountPath: /var/hyperledger - - name: fabric-config - mountPath: /var/hyperledger/fabric/config - - # This init container will unfurl all of the MSP archives listed in the msp-config config map. - initContainers: - - name: msp-unfurl - image: busybox - command: - - sh - - -c - - "for msp in $(ls /msp/msp-*.tgz); do echo $msp && tar zxvf $msp -C /var/hyperledger/fabric; done" - volumeMounts: - - name: msp-config - mountPath: /msp - - name: fabric-volume - mountPath: /var/hyperledger - - volumes: - - name: fabric-volume - persistentVolumeClaim: - claimName: fabric-org2 - - name: fabric-config - configMap: - name: org2-config - - name: msp-config - configMap: - name: msp-config diff --git a/test-network-k8s/kube/org2/org2-ca.yaml b/test-network-k8s/kube/org2/org2-ca.yaml index ef8c030f..59f7c000 100644 --- a/test-network-k8s/kube/org2/org2-ca.yaml +++ b/test-network-k8s/kube/org2/org2-ca.yaml @@ -17,6 +17,7 @@ spec: - localhost - org2-ca - org2-ca.test-network.svc.cluster.local + - org2-ca.${DOMAIN} ipAddresses: - 127.0.0.1 secretName: org2-ca-tls-cert @@ -40,7 +41,7 @@ spec: spec: containers: - name: main - image: {{FABRIC_CONTAINER_REGISTRY}}/fabric-ca:{{FABRIC_CA_VERSION}} + image: ${FABRIC_CONTAINER_REGISTRY}/fabric-ca:${FABRIC_CA_VERSION} imagePullPolicy: IfNotPresent env: - name: FABRIC_CA_SERVER_CA_NAME @@ -88,8 +89,36 @@ metadata: name: org2-ca spec: ports: - - name: tls + - name: https port: 443 protocol: TCP selector: - app: org2-ca \ No newline at end of file + app: org2-ca + +--- +apiVersion: networking.k8s.io/v1 +kind: Ingress +metadata: + annotations: + nginx.ingress.kubernetes.io/proxy-connect-timeout: 60s + nginx.ingress.kubernetes.io/ssl-passthrough: "true" + labels: + app: org2-ca + name: org2-ca +spec: + ingressClassName: nginx + rules: + - host: org2-ca.${DOMAIN} + http: + paths: + - backend: + service: + name: org2-ca + port: + name: https + path: / + pathType: ImplementationSpecific + tls: + - hosts: + - org2-ca.${DOMAIN} + \ No newline at end of file diff --git a/test-network-k8s/kube/org2/org2-peer1.yaml b/test-network-k8s/kube/org2/org2-peer1.yaml index f7383325..89e3440b 100644 --- a/test-network-k8s/kube/org2/org2-peer1.yaml +++ b/test-network-k8s/kube/org2/org2-peer1.yaml @@ -17,8 +17,10 @@ spec: dnsNames: - localhost - org2-peer1 - - org2-peer-gateway-svc - org2-peer1.test-network.svc.cluster.local + - org2-peer1.${DOMAIN} + - org2-peer-gateway-svc + - org2-peer-gateway-svc.${DOMAIN} ipAddresses: - 127.0.0.1 secretName: org2-peer1-tls-cert @@ -74,7 +76,7 @@ spec: spec: containers: - name: main - image: {{FABRIC_CONTAINER_REGISTRY}}/fabric-peer:{{FABRIC_VERSION}} + image: ${FABRIC_CONTAINER_REGISTRY}/fabric-peer:${FABRIC_VERSION} imagePullPolicy: IfNotPresent envFrom: - configMapRef: @@ -118,7 +120,7 @@ metadata: name: org2-peer1 spec: ports: - - name: gossip + - name: grpc port: 7051 protocol: TCP - name: chaincode @@ -136,8 +138,46 @@ metadata: name: org2-peer-gateway-svc spec: ports: - - name: gossip + - name: grpc port: 7051 protocol: TCP selector: org: org2 +--- +apiVersion: networking.k8s.io/v1 +kind: Ingress +metadata: + annotations: + nginx.ingress.kubernetes.io/proxy-connect-timeout: 60s + nginx.ingress.kubernetes.io/ssl-passthrough: "true" + labels: + app: org2-peer1 + name: org2-peer1 +spec: + ingressClassName: nginx + rules: + - host: org2-peer1.${DOMAIN} + http: + paths: + - backend: + service: + name: org2-peer1 + port: + name: grpc + path: / + pathType: ImplementationSpecific + - host: org2-peer-gateway-svc.${DOMAIN} + http: + paths: + - backend: + service: + name: org2-peer1 + port: + name: grpc + path: / + pathType: ImplementationSpecific + tls: + - hosts: + - org2-peer1.${DOMAIN} + - hosts: + - org2-peer-gateway-svc.${DOMAIN} diff --git a/test-network-k8s/kube/org2/org2-peer2.yaml b/test-network-k8s/kube/org2/org2-peer2.yaml index 15bddd50..268dde28 100644 --- a/test-network-k8s/kube/org2/org2-peer2.yaml +++ b/test-network-k8s/kube/org2/org2-peer2.yaml @@ -19,6 +19,7 @@ spec: - org2-peer2 - org2-peer-gateway-svc - org2-peer2.test-network.svc.cluster.local + - org2-peer2.${DOMAIN} ipAddresses: - 127.0.0.1 secretName: org2-peer2-tls-cert @@ -74,7 +75,7 @@ spec: spec: containers: - name: main - image: {{FABRIC_CONTAINER_REGISTRY}}/fabric-peer:{{FABRIC_VERSION}} + image: ${FABRIC_CONTAINER_REGISTRY}/fabric-peer:${FABRIC_VERSION} imagePullPolicy: IfNotPresent envFrom: - configMapRef: @@ -118,7 +119,7 @@ metadata: name: org2-peer2 spec: ports: - - name: gossip + - name: grpc port: 7051 protocol: TCP - name: chaincode @@ -129,3 +130,29 @@ spec: protocol: TCP selector: app: org2-peer2 +--- +apiVersion: networking.k8s.io/v1 +kind: Ingress +metadata: + annotations: + nginx.ingress.kubernetes.io/proxy-connect-timeout: 60s + nginx.ingress.kubernetes.io/ssl-passthrough: "true" + labels: + app: org2-peer2 + name: org2-peer2 +spec: + ingressClassName: nginx + rules: + - host: org2-peer2.${DOMAIN} + http: + paths: + - backend: + service: + name: org2-peer2 + port: + name: grpc + path: / + pathType: ImplementationSpecific + tls: + - hosts: + - org2-peer2.${DOMAIN} \ No newline at end of file diff --git a/test-network-k8s/network b/test-network-k8s/network index b53b0634..955aef67 100755 --- a/test-network-k8s/network +++ b/test-network-k8s/network @@ -6,28 +6,25 @@ # set -o errexit -# todo: better handling for input parameters. +# todo: better handling for input parameters. Argbash? # todo: skip storage volume init if deploying to a remote / cloud cluster (ICP IKS ROKS etc...) # todo: for logging, set up a stack and allow multi-line status output codes -# todo: refactor - lots of for-org-in-0-to-2-... -# todo: find a better technique for passing input commands to a remote kube exec -# todo: register tls csr.hosts w/ kube DNS domain .NS.svc.cluster.local # todo: user:pass auth for tls and ecert bootstrap admins. here and in the server-config.yaml -# todo: set tls.certfiles= ... arg in deployment env / yaml # todo: refactor chaincode install to support other chaincode routines -# todo: consider using templates for boilerplate network nodes (orderers, peers, ...) # todo: allow the user to specify the chaincode name (hardcoded as 'basic') both in install and invoke/query # todo: track down a nasty bug whereby the CA service endpoints (kube services) will occasionally reject TCP connections after network down/up. This is patched by introducing a 10s sleep after the deployments are up... -# todo: refactor query/invoke to specify chaincode name (-n param) -CONTAINER_CLI=${CONTAINER_CLI:-docker} -FABRIC_VERSION=${TEST_NETWORK_FABRIC_VERSION:-2.4} -FABRIC_CA_VERSION=${TEST_NETWORK_FABRIC_CA_VERSION:-1.5} -FABRIC_CONTAINER_REGISTRY=${TEST_NETWORK_FABRIC_CONTAINER_REGISTRY:-hyperledger} -NETWORK_NAME=${TEST_NETWORK_NAME:-test-network} -CLUSTER_NAME=${TEST_NETWORK_KIND_CLUSTER_NAME:-kind} -NS=${TEST_NETWORK_KUBE_NAMESPACE:-${NETWORK_NAME}} -CHANNEL_NAME=${TEST_NETWORK_CHANNEL_NAME:-mychannel} +export CONTAINER_CLI=${CONTAINER_CLI:-docker} +export FABRIC_VERSION=${TEST_NETWORK_FABRIC_VERSION:-2.4.3} +export FABRIC_CA_VERSION=${TEST_NETWORK_FABRIC_CA_VERSION:-1.5.2} +export FABRIC_CONTAINER_REGISTRY=${TEST_NETWORK_FABRIC_CONTAINER_REGISTRY:-hyperledger} +export NETWORK_NAME=${TEST_NETWORK_NAME:-test-network} +export CLUSTER_NAME=${TEST_NETWORK_KIND_CLUSTER_NAME:-kind} +export NS=${TEST_NETWORK_KUBE_NAMESPACE:-${NETWORK_NAME}} +export DOMAIN=${TEST_NETWORK_DOMAIN:-vcap.me} +export CHANNEL_NAME=${TEST_NETWORK_CHANNEL_NAME:-mychannel} +export TEMP_DIR=${PWD}/build + LOG_FILE=${TEST_NETWORK_LOG_FILE:-network.log} DEBUG_FILE=${TEST_NETWORK_DEBUG_FILE:-network-debug.log} LOCAL_REGISTRY_NAME=${TEST_NETWORK_LOCAL_REGISTRY_NAME:-kind-registry} @@ -50,6 +47,7 @@ function print_help() { log "Fabric CA Version \t: ${FABRIC_CA_VERSION}" log "Container Registry \t: ${FABRIC_CONTAINER_REGISTRY}" log "Network name \t\t: ${NETWORK_NAME}" + log "Ingress domain \t\t: ${DOMAIN}" log "Channel name \t\t: ${CHANNEL_NAME}" log log "--- Chaincode Information" @@ -71,8 +69,6 @@ function print_help() { log "Debug log file \t\t: ${DEBUG_FILE}" log - - echo todo: help output, parse mode, flags, env, etc. } diff --git a/test-network-k8s/scripts/chaincode.sh b/test-network-k8s/scripts/chaincode.sh index 5eee57e7..e1275609 100755 --- a/test-network-k8s/scripts/chaincode.sh +++ b/test-network-k8s/scripts/chaincode.sh @@ -22,37 +22,23 @@ function package_chaincode_for() { pop_fn } -# Copy the chaincode archive from the local host to the org admin -function transfer_chaincode_archive_for() { - local org=$1 - local cc_archive="build/chaincode/${CHAINCODE_NAME}.tgz" - push_fn "Transferring chaincode archive to ${org}" - - # Like kubectl cp, but targeted to a deployment rather than an individual pod. - tar cf - ${cc_archive} | kubectl -n $NS exec -i deploy/${org}-admin-cli -c main -- tar xvf - - - pop_fn -} - function install_chaincode_for() { local org=$1 local peer=$2 - push_fn "Installing chaincode for org ${org} peer ${peer}" + push_fn "Installing chaincode for org ${org} peer ${peer}" - # Install the chaincode - echo 'set -x - export CORE_PEER_ADDRESS='${org}'-'${peer}':7051 - peer lifecycle chaincode install build/chaincode/'${CHAINCODE_NAME}'.tgz - ' | exec kubectl -n $NS exec deploy/${org}-admin-cli -c main -i -- /bin/bash + export_peer_context $org $peer + + peer lifecycle chaincode install build/chaincode/${CHAINCODE_NAME}.tgz pop_fn } function launch_chaincode_service() { local org=$1 - local cc_id=$2 - local cc_image=$3 - local peer=$4 + local peer=$2 + local cc_id=$3 + local cc_image=$4 push_fn "Launching chaincode container \"${cc_image}\"" # The chaincode endpoint needs to have the generated chaincode ID available in the environment. @@ -72,76 +58,72 @@ function launch_chaincode_service() { function activate_chaincode_for() { local org=$1 - local cc_id=$2 - push_fn "Activating chaincode ${CHAINCODE_ID}" + local peer=$2 + local cc_id=$3 + push_fn "Activating $org chaincode ${CHAINCODE_ID}" + + export_peer_context $org $peer - echo 'set -x - export CORE_PEER_ADDRESS='${org}'-peer1:7051 - peer lifecycle \ chaincode approveformyorg \ - --channelID '${CHANNEL_NAME}' \ - --name '${CHAINCODE_NAME}' \ - --version 1 \ - --package-id '${cc_id}' \ - --sequence 1 \ - -o org0-orderer1:6050 \ - --tls --cafile /var/hyperledger/fabric/organizations/ordererOrganizations/org0.example.com/msp/tlscacerts/org0-tls-ca.pem - + --channelID ${CHANNEL_NAME} \ + --name ${CHAINCODE_NAME} \ + --version 1 \ + --package-id ${cc_id} \ + --sequence 1 \ + --orderer org0-orderer1.${DOMAIN}:443 \ + --tls --cafile ${TEMP_DIR}/channel-msp/ordererOrganizations/org0/orderers/org0-orderer1/tls/signcerts/tls-cert.pem + peer lifecycle \ chaincode commit \ - --channelID '${CHANNEL_NAME}' \ - --name '${CHAINCODE_NAME}' \ - --version 1 \ - --sequence 1 \ - -o org0-orderer1:6050 \ - --tls --cafile /var/hyperledger/fabric/organizations/ordererOrganizations/org0.example.com/msp/tlscacerts/org0-tls-ca.pem - ' | exec kubectl -n $NS exec deploy/${org}-admin-cli -c main -i -- /bin/bash + --channelID ${CHANNEL_NAME} \ + --name ${CHAINCODE_NAME} \ + --version 1 \ + --sequence 1 \ + --orderer org0-orderer1.${DOMAIN}:443 \ + --tls --cafile ${TEMP_DIR}/channel-msp/ordererOrganizations/org0/orderers/org0-orderer1/tls/signcerts/tls-cert.pem pop_fn } function query_chaincode() { set -x - # todo: mangle additional $@ parameters with bash escape quotations - echo ' - export CORE_PEER_ADDRESS=org1-peer1:7051 - peer chaincode query -n '${CHAINCODE_NAME}' -C '${CHANNEL_NAME}' -c '"'$@'"' - ' | exec kubectl -n $NS exec deploy/org1-admin-cli -c main -i -- /bin/bash + + export_peer_context org1 peer1 + + peer chaincode query \ + -n $CHAINCODE_NAME \ + -C $CHANNEL_NAME \ + -c $@ } function query_chaincode_metadata() { set -x local args='{"Args":["org.hyperledger.fabric:GetMetadata"]}' - # todo: mangle additional $@ parameters with bash escape quotations + + log '' log 'Org1-Peer1:' - echo ' - export CORE_PEER_ADDRESS=org1-peer1:7051 - peer chaincode query -n '${CHAINCODE_NAME}' -C '${CHANNEL_NAME}' -c '"'$args'"' - ' | exec kubectl -n $NS exec deploy/org1-admin-cli -c main -i -- /bin/bash + export_peer_context org1 peer1 + peer chaincode query -n $CHAINCODE_NAME -C $CHANNEL_NAME -c $args log '' log 'Org1-Peer2:' - echo ' - export CORE_PEER_ADDRESS=org1-peer2:7051 - peer chaincode query -n '${CHAINCODE_NAME}' -C '${CHANNEL_NAME}' -c '"'$args'"' - ' | exec kubectl -n $NS exec deploy/org1-admin-cli -c main -i -- /bin/bash - + export_peer_context org1 peer2 + peer chaincode query -n $CHAINCODE_NAME -C $CHANNEL_NAME -c $args } function invoke_chaincode() { # set -x # todo: mangle additional $@ parameters with bash escape quotations - echo ' - export CORE_PEER_ADDRESS=org1-peer1:7051 - peer chaincode \ - invoke \ - -o org0-orderer1:6050 \ - --tls --cafile /var/hyperledger/fabric/organizations/ordererOrganizations/org0.example.com/msp/tlscacerts/org0-tls-ca.pem \ - -n '${CHAINCODE_NAME}' \ - -C '${CHANNEL_NAME}' \ - -c '"'$@'"' - ' | exec kubectl -n $NS exec deploy/org1-admin-cli -c main -i -- /bin/bash + + export_peer_context org1 peer1 + + peer chaincode invoke \ + -n $CHAINCODE_NAME \ + -C $CHANNEL_NAME \ + -c $@ \ + --orderer org0-orderer1.${DOMAIN}:443 \ + --tls --cafile ${TEMP_DIR}/channel-msp/ordererOrganizations/org0/orderers/org0-orderer1/tls/signcerts/tls-cert.pem sleep 2 } @@ -162,7 +144,7 @@ function install_chaincode() { local org=org1 package_chaincode_for ${org} - transfer_chaincode_archive_for ${org} + install_chaincode_for ${org} peer1 install_chaincode_for ${org} peer2 @@ -174,7 +156,11 @@ function activate_chaincode() { set -x set_chaincode_id - activate_chaincode_for org1 $CHAINCODE_ID + activate_chaincode_for org1 peer1 $CHAINCODE_ID + + # jdk: does activation on a single peer apply to all peers in the org? This is an error: +# activate_chaincode_for org1 peer1 $CHAINCODE_ID + } # Install, launch, and activate the chaincode @@ -182,8 +168,8 @@ function deploy_chaincode() { set -x install_chaincode - launch_chaincode_service org1 $CHAINCODE_ID $CHAINCODE_IMAGE peer1 - launch_chaincode_service org1 $CHAINCODE_ID $CHAINCODE_IMAGE peer2 + launch_chaincode_service org1 peer1 $CHAINCODE_ID $CHAINCODE_IMAGE + launch_chaincode_service org1 peer2 $CHAINCODE_ID $CHAINCODE_IMAGE activate_chaincode } diff --git a/test-network-k8s/scripts/channel.sh b/test-network-k8s/scripts/channel.sh old mode 100755 new mode 100644 index e9eb18d2..d5d4e7c9 --- a/test-network-k8s/scripts/channel.sh +++ b/test-network-k8s/scripts/channel.sh @@ -5,197 +5,267 @@ # SPDX-License-Identifier: Apache-2.0 # -function create_channel_org_MSP() { +function channel_up() { + set -x + + register_org_admins + enroll_org_admins + + create_channel_MSP + create_genesis_block + + join_channel_orderers + join_channel_peers +} + +function register_org_admins() { + push_fn "Registering org Admin users" + + register_org_admin org0 org0admin org0adminpw + register_org_admin org1 org1admin org1adminpw + register_org_admin org2 org2admin org2adminpw + + pop_fn +} + +# Register the org admin user +function register_org_admin() { + local type=admin local org=$1 - local org_type=$2 - local ecert_ca=${org}-ca - - echo 'set -x - - mkdir -p /var/hyperledger/fabric/organizations/'${org_type}'Organizations/'${org}'.example.com/msp/cacerts - cp \ - $FABRIC_CA_CLIENT_HOME/'${ecert_ca}'/rcaadmin/msp/cacerts/'${ecert_ca}'.pem \ - /var/hyperledger/fabric/organizations/'${org_type}'Organizations/'${org}'.example.com/msp/cacerts + local id_name=$2 + local id_secret=$3 + local ca_name=${org}-ca - mkdir -p /var/hyperledger/fabric/organizations/'${org_type}'Organizations/'${org}'.example.com/msp/tlscacerts - cp \ - /var/hyperledger/fabric/config/tls/ca.crt \ - /var/hyperledger/fabric/organizations/'${org_type}'Organizations/'${org}'.example.com/msp/tlscacerts/'${org}'-tls-ca.pem + echo "Registering org admin $username" - echo "NodeOUs: - Enable: true - ClientOUIdentifier: - Certificate: cacerts/'${ecert_ca}'.pem - OrganizationalUnitIdentifier: client - PeerOUIdentifier: - Certificate: cacerts/'${ecert_ca}'.pem - OrganizationalUnitIdentifier: peer - AdminOUIdentifier: - Certificate: cacerts/'${ecert_ca}'.pem - OrganizationalUnitIdentifier: admin - OrdererOUIdentifier: - Certificate: cacerts/'${ecert_ca}'.pem - OrganizationalUnitIdentifier: orderer "> /var/hyperledger/fabric/organizations/'${org_type}'Organizations/'${org}'.example.com/msp/config.yaml - - ' | exec kubectl -n $NS exec deploy/${ecert_ca} -i -- /bin/sh + cat < ${CA_DIR}/tls-cert.pem + + # enroll the org admin + FABRIC_CA_CLIENT_HOME=${ORG_ADMIN_DIR} fabric-ca-client enroll \ + --url ${CA_URL} \ + --tls.certfiles ${CA_DIR}/tls-cert.pem + + # Construct an msp config.yaml + CA_CERT_NAME=${CA_NAME}-$(echo $DOMAIN | tr -s . -)-${CA_PORT}.pem + + create_msp_config_yaml ${CA_NAME} ${CA_CERT_NAME} ${ORG_ADMIN_DIR}/msp + + # private keys are hashed by name, but we only support one enrollment. + # test-network examples refer to this as "server.key", which is incorrect. + # This is the private key used to endorse transactions using the admin's + # public key. + mv ${ORG_ADMIN_DIR}/msp/keystore/*_sk ${ORG_ADMIN_DIR}/msp/keystore/key.pem +} + +# create an enrollment MSP config.yaml +function create_msp_config_yaml() { + local ca_name=$1 + local ca_cert_name=$2 + local msp_dir=$3 + echo "Creating msp config ${msp_dir}/config.yaml with cert ${ca_cert_name}" + + cat << EOF > ${msp_dir}/config.yaml +NodeOUs: + Enable: true + ClientOUIdentifier: + Certificate: cacerts/${ca_cert_name} + OrganizationalUnitIdentifier: client + PeerOUIdentifier: + Certificate: cacerts/${ca_cert_name} + OrganizationalUnitIdentifier: peer + AdminOUIdentifier: + Certificate: cacerts/${ca_cert_name} + OrganizationalUnitIdentifier: admin + OrdererOUIdentifier: + Certificate: cacerts/${ca_cert_name} + OrganizationalUnitIdentifier: orderer +EOF } function create_channel_MSP() { push_fn "Creating channel MSP" - create_channel_org_MSP org0 orderer + create_channel_org_MSP org0 orderer create_channel_org_MSP org1 peer create_channel_org_MSP org2 peer - pop_fn -} - -function aggregate_channel_MSP() { - push_fn "Aggregating channel MSP" - - rm -rf ./build/msp/ - mkdir -p ./build/msp - - kubectl -n $NS exec deploy/org0-ca -- tar zcvf - -C /var/hyperledger/fabric organizations/ordererOrganizations/org0.example.com/msp > build/msp/msp-org0.example.com.tgz - kubectl -n $NS exec deploy/org1-ca -- tar zcvf - -C /var/hyperledger/fabric organizations/peerOrganizations/org1.example.com/msp > build/msp/msp-org1.example.com.tgz - kubectl -n $NS exec deploy/org2-ca -- tar zcvf - -C /var/hyperledger/fabric organizations/peerOrganizations/org2.example.com/msp > build/msp/msp-org2.example.com.tgz - - kubectl -n $NS delete configmap msp-config || true - kubectl -n $NS create configmap msp-config --from-file=build/msp/ + extract_orderer_tls_cert org0 orderer1 + extract_orderer_tls_cert org0 orderer2 + extract_orderer_tls_cert org0 orderer3 pop_fn } -function launch_admin_CLIs() { - push_fn "Launching admin CLIs" +function create_channel_org_MSP() { + local org=$1 + local type=$2 + local ca_name=${org}-ca - launch kube/org0/org0-admin-cli.yaml - launch kube/org1/org1-admin-cli.yaml - launch kube/org2/org2-admin-cli.yaml + ORG_MSP_DIR=${TEMP_DIR}/channel-msp/${type}Organizations/${org}/msp + mkdir -p ${ORG_MSP_DIR}/cacerts + mkdir -p ${ORG_MSP_DIR}/tlscacerts - kubectl -n $NS rollout status deploy/org0-admin-cli - kubectl -n $NS rollout status deploy/org1-admin-cli - kubectl -n $NS rollout status deploy/org2-admin-cli + # extract the CA's signing authority from the CA/cainfo response + curl -s \ + --cacert ${TEMP_DIR}/cas/${ca_name}/tls-cert.pem \ + https://${ca_name}.${DOMAIN}/cainfo \ + | jq -r .result.CAChain \ + | base64 -d \ + > ${ORG_MSP_DIR}/cacerts/ca-signcert.pem - pop_fn + # extract the CA's TLS CA certificate from the cert-manager secret + kubectl -n $NS get secret ${ca_name}-tls-cert -o json \ + | jq -r .data.\"ca.crt\" \ + | base64 -d \ + > ${ORG_MSP_DIR}/tlscacerts/tlsca-signcert.pem + + # create an MSP config.yaml with the CA's signing certificate + create_msp_config_yaml ${ca_name} ca-signcert.pem ${ORG_MSP_DIR} +} + +# Extract an orderer's TLS signing certificate for inclusion in the channel config block +function extract_orderer_tls_cert() { + local org=$1 + local orderer=$2 + + echo "Extracting TLS cert for $org $orderer" + + ORDERER_TLS_DIR=${TEMP_DIR}/channel-msp/ordererOrganizations/${org}/orderers/${org}-${orderer}/tls + mkdir -p $ORDERER_TLS_DIR/signcerts + + kubectl -n $NS get secret ${org}-${orderer}-tls-cert -o json \ + | jq -r .data.\"tls.crt\" \ + | base64 -d \ + > ${ORDERER_TLS_DIR}/signcerts/tls-cert.pem } function create_genesis_block() { - push_fn "Creating channel \"${CHANNEL_NAME}\"" + push_fn "Creating channel genesis block" - echo 'set -x - configtxgen -profile TwoOrgsApplicationGenesis -channelID '${CHANNEL_NAME}' -outputBlock genesis_block.pb - # configtxgen -inspectBlock genesis_block.pb - - osnadmin channel join --orderer-address org0-orderer1:9443 --channelID '${CHANNEL_NAME}' --config-block genesis_block.pb - osnadmin channel join --orderer-address org0-orderer2:9443 --channelID '${CHANNEL_NAME}' --config-block genesis_block.pb - osnadmin channel join --orderer-address org0-orderer3:9443 --channelID '${CHANNEL_NAME}' --config-block genesis_block.pb - - ' | exec kubectl -n $NS exec deploy/org0-admin-cli -i -- /bin/bash - - # todo: readiness / liveiness equivalent for channel ? Needs a little bit to settle before peers can join. + FABRIC_CFG_PATH=${PWD}/config/org0 \ + configtxgen \ + -profile TwoOrgsApplicationGenesis \ + -channelID $CHANNEL_NAME \ + -outputBlock ${TEMP_DIR}/genesis_block.pb + + # configtxgen -inspectBlock ${TEMP_DIR}/genesis_block.pb + + pop_fn +} + +function join_channel_orderers() { + push_fn "Joining orderers to channel ${CHANNEL_NAME}" + + join_channel_orderer org0 orderer1 + join_channel_orderer org0 orderer2 + join_channel_orderer org0 orderer3 + + # todo: readiness / liveiness equivalent for channel? Needs a little bit to settle before peers can join. sleep 10 pop_fn } -function join_org_peers() { +# Request from the channel ADMIN api that the orderer joins the target channel +function join_channel_orderer() { local org=$1 - push_fn "Joining ${org} peers to channel \"${CHANNEL_NAME}\"" + local orderer=$2 - echo 'set -x - # Fetch the genesis block from an orderer - peer channel \ - fetch oldest \ - genesis_block.pb \ - -c '${CHANNEL_NAME}' \ - -o org0-orderer1:6050 \ - --tls --cafile /var/hyperledger/fabric/organizations/ordererOrganizations/org0.example.com/msp/tlscacerts/org0-tls-ca.pem - - # Join peer1 to the channel. - CORE_PEER_ADDRESS='${org}'-peer1:7051 \ - peer channel \ - join \ - -b genesis_block.pb \ - -o org0-orderer1:6050 \ - --tls --cafile /var/hyperledger/fabric/organizations/ordererOrganizations/org0.example.com/msp/tlscacerts/org0-tls-ca.pem - - # Join peer2 to the channel. - CORE_PEER_ADDRESS='${org}'-peer2:7051 \ - peer channel \ - join \ - -b genesis_block.pb \ - -o org0-orderer1:6050 \ - --tls --cafile /var/hyperledger/fabric/organizations/ordererOrganizations/org0.example.com/msp/tlscacerts/org0-tls-ca.pem - - ' | exec kubectl -n $NS exec deploy/${org}-admin-cli -i -- /bin/bash - - pop_fn + # The client certificate presented in this case is the admin user's enrollment key. This is a stronger assertion + # of identity than the Docker Compose network, which transmits the orderer node's TLS key pair directly + osnadmin channel join \ + --orderer-address ${org}-${orderer}-admin.${DOMAIN} \ + --ca-file ${TEMP_DIR}/channel-msp/ordererOrganizations/${org}/orderers/${org}-${orderer}/tls/signcerts/tls-cert.pem \ + --client-cert ${TEMP_DIR}/enrollments/${org}/users/${org}admin/msp/signcerts/cert.pem \ + --client-key ${TEMP_DIR}/enrollments/${org}/users/${org}admin/msp/keystore/key.pem \ + --channelID ${CHANNEL_NAME} \ + --config-block ${TEMP_DIR}/genesis_block.pb } -function join_peers() { +function join_channel_peers() { join_org_peers org1 join_org_peers org2 } -# Copy the scripts/anchor_peers.sh to a remote volume -function push_anchor_peer_script() { +function join_org_peers() { local org=$1 + push_fn "Joining ${org} peers to channel ${CHANNEL_NAME}" - tar cf - scripts/ | kubectl -n $NS exec -i -c main deploy/${org}-admin-cli -- tar xf - -C /var/hyperledger/fabric -} - -verify_result() { - if [ $1 -ne 0 ]; then - echo $2 - exit $1 - fi -} - -# Launch the anchor peer update script on a remote org admin CLI -function invoke_anchor_peer_update() { - local org_num=$1 - local peer_name=$2 - - kubectl exec \ - -n $NS \ - -c main \ - deploy/org${org_num}-admin-cli \ - -i \ - /bin/bash -c "/var/hyperledger/fabric/scripts/set_anchor_peer.sh ${org_num} ${CHANNEL_NAME} ${peer_name}" - - verify_result $? "Error updating anchor peer for org ${org_num}" -} - -# -# To update the anchor peers we will need to execute a script on each of the peer admin CLI containers. These -# commands can be individually piped into kubectl exec ... but it will be simpler if we transfer the anchor -# peer update script over to the org volume and then trigger it from kubectl. -# -function update_anchor_peers() { - local peer_name=$1 - push_fn "Updating anchor peers to ${peer_name}" - - push_anchor_peer_script org1 - push_anchor_peer_script org2 - - invoke_anchor_peer_update 1 ${peer_name} - invoke_anchor_peer_update 2 ${peer_name} + # Join peers to channel + join_channel_peer $org peer1 + join_channel_peer $org peer2 pop_fn } -function channel_up() { +function join_channel_peer() { + local org=$1 + local peer=$2 - create_channel_MSP - aggregate_channel_MSP - launch_admin_CLIs + export_peer_context $org $peer - create_genesis_block - join_peers - - # peer1 was set as the anchor peer in configtx.yaml. Setting this again will force an - # error to be returned from the channel up. We might want to render the warning in - # this case to indicate that the call was made but had a nonzero exit. - # update_anchor_peers peer1 -} \ No newline at end of file + peer channel join \ + --blockpath ${TEMP_DIR}/genesis_block.pb \ + --orderer org0-orderer1.${DOMAIN} \ + --tls \ + --cafile ${TEMP_DIR}/channel-msp/ordererOrganizations/org0/orderers/org0-orderer1/tls/signcerts/tls-cert.pem +} diff --git a/test-network-k8s/scripts/fabric_CAs.sh b/test-network-k8s/scripts/fabric_CAs.sh index a9824971..fd31b555 100755 --- a/test-network-k8s/scripts/fabric_CAs.sh +++ b/test-network-k8s/scripts/fabric_CAs.sh @@ -5,27 +5,19 @@ # SPDX-License-Identifier: Apache-2.0 # -function launch_CA() { - local yaml=$1 - cat ${yaml} \ - | sed 's,{{FABRIC_CONTAINER_REGISTRY}},'${FABRIC_CONTAINER_REGISTRY}',g' \ - | sed 's,{{FABRIC_CA_VERSION}},'${FABRIC_CA_VERSION}',g' \ - | kubectl -n $NS apply -f - -} - function launch_ECert_CAs() { push_fn "Launching Fabric CAs" - launch_CA kube/org0/org0-ca.yaml - launch_CA kube/org1/org1-ca.yaml - launch_CA kube/org2/org2-ca.yaml + apply_template kube/org0/org0-ca.yaml + apply_template kube/org1/org1-ca.yaml + apply_template kube/org2/org2-ca.yaml kubectl -n $NS rollout status deploy/org0-ca kubectl -n $NS rollout status deploy/org1-ca kubectl -n $NS rollout status deploy/org2-ca # todo: this papers over a nasty bug whereby the CAs are ready, but sporadically refuse connections after a down / up - # sleep 10 + sleep 5 pop_fn } diff --git a/test-network-k8s/scripts/kind.sh b/test-network-k8s/scripts/kind.sh index 02cb9165..14cd6279 100755 --- a/test-network-k8s/scripts/kind.sh +++ b/test-network-k8s/scripts/kind.sh @@ -31,7 +31,7 @@ function load_docker_images() { } function apply_nginx_ingress() { - push_fn "Launching Nginx ingress controller" + push_fn "Launching ingress controller" # This ingress-nginx.yaml was generated 9/24 from https://raw.githubusercontent.com/kubernetes/ingress-nginx/main/deploy/static/provider/kind/deploy.yaml # with modifications for ssl-passthrough required to launch IBP-support with the nginx ingress. @@ -42,12 +42,29 @@ function apply_nginx_ingress() { pop_fn } -function install_cert_manager() { - push_fn "Installing cert-manager" +function wait_for_nginx_ingress() { + push_fn "Waiting for ingress controller" - # Install cert-manager to manage TLS certificates + kubectl wait --namespace ingress-nginx \ + --for=condition=ready pod \ + --selector=app.kubernetes.io/component=controller \ + --timeout=90s + + pop_fn +} + +function apply_cert_manager() { + push_fn "Launching cert-manager" + + # Install cert-manager to manage TLS certificates kubectl apply -f https://github.com/jetstack/cert-manager/releases/download/v1.6.1/cert-manager.yaml + pop_fn +} + +function wait_for_cert_manager() { + push_fn "Waiting for cert-manager" + kubectl -n cert-manager rollout status deploy/cert-manager kubectl -n cert-manager rollout status deploy/cert-manager-cainjector kubectl -n cert-manager rollout status deploy/cert-manager-webhook @@ -58,6 +75,9 @@ function install_cert_manager() { function kind_create() { push_fn "Creating cluster \"${CLUSTER_NAME}\"" + # prevent the next kind cluster from using the previous Fabric network's enrollments. + rm -rf $PWD/build + # todo: always delete? Maybe return no-op if the cluster already exists? kind delete cluster --name $CLUSTER_NAME @@ -87,8 +107,8 @@ nodes: - containerPort: 443 hostPort: ${ingress_https_port} protocol: TCP -networking: - kubeProxyMode: "ipvs" +#networking: +# kubeProxyMode: "ipvs" # create a cluster with the local registry enabled in containerd containerdConfigPatches: @@ -98,6 +118,12 @@ containerdConfigPatches: EOF + # workaround for https://github.com/hyperledger/fabric-samples/issues/550 - pods can not resolve external DNS + for node in $(kind get nodes); + do + docker exec "$node" sysctl net.ipv4.conf.all.route_localnet=1; + done + pop_fn } @@ -151,13 +177,16 @@ function kind_init() { kind_create apply_nginx_ingress - install_cert_manager + apply_cert_manager launch_docker_registry if [ "${STAGE_DOCKER_IMAGES}" == true ]; then pull_docker_images load_docker_images - fi + fi + + wait_for_cert_manager + wait_for_nginx_ingress } function kind_unkind() { diff --git a/test-network-k8s/scripts/prereqs.sh b/test-network-k8s/scripts/prereqs.sh index bdef9d91..7fa00262 100755 --- a/test-network-k8s/scripts/prereqs.sh +++ b/test-network-k8s/scripts/prereqs.sh @@ -8,6 +8,8 @@ # Double check that kind, kubectl, docker, and all required images are present. function check_prereqs() { + set +e + ${CONTAINER_CLI} version > /dev/null if [[ $? -ne 0 ]]; then echo "No '${CONTAINER_CLI}' binary available?" @@ -31,4 +33,27 @@ function check_prereqs() { echo "No 'jq' binary available? (https://stedolan.github.io/jq/)" exit 1 fi + + # Use the local fabric binaries if available. If not, go get them. + bin/peer version &> /dev/null + if [[ $? -ne 0 ]]; then + echo "Downloading LATEST Fabric binaries and config" + curl -sSL https://raw.githubusercontent.com/hyperledger/fabric/main/scripts/bootstrap.sh | bash -s -- -s -d + + # remove sample config files extracted by the installation script + rm config/configtx.yaml + rm config/core.yaml + rm config/orderer.yaml + fi + + export PATH=bin:$PATH + + # Double-check that the binary transfer was OK + peer version > /dev/null + if [[ $? -ne 0 ]]; then + log "No 'peer' binary available?" + exit 1 + fi + + set -e } \ No newline at end of file diff --git a/test-network-k8s/scripts/rest_sample.sh b/test-network-k8s/scripts/rest_sample.sh index 9888f589..c40c6b78 100755 --- a/test-network-k8s/scripts/rest_sample.sh +++ b/test-network-k8s/scripts/rest_sample.sh @@ -5,16 +5,8 @@ # SPDX-License-Identifier: Apache-2.0 # -function extract_MSP_archives() { - mkdir -p build/msp - - kubectl -n $NS exec deploy/org1-ca -- tar zcf - -C /var/hyperledger/fabric organizations/peerOrganizations/org1.example.com/msp | tar zxf - -C build/msp - kubectl -n $NS exec deploy/org2-ca -- tar zcf - -C /var/hyperledger/fabric organizations/peerOrganizations/org2.example.com/msp | tar zxf - -C build/msp - - kubectl -n $NS exec deploy/org1-ca -- tar zcf - -C /var/hyperledger/fabric organizations/peerOrganizations/org1.example.com/users/Admin@org1.example.com/msp | tar zxf - -C build/msp - kubectl -n $NS exec deploy/org2-ca -- tar zcf - -C /var/hyperledger/fabric organizations/peerOrganizations/org2.example.com/users/Admin@org2.example.com/msp | tar zxf - -C build/msp -} - +# This magical awk script led to 30 hours of debugging a "TLS handshake error" +# moral: do not edit / alter the number of '\' in the following transform: function one_line_pem { echo "`awk 'NF {sub(/\\n/, ""); printf "%s\\\\\\\n",$0;}' $1`" } @@ -32,41 +24,32 @@ function json_ccp { function construct_rest_sample_configmap() { push_fn "Constructing fabric-rest-sample connection profiles" - extract_MSP_archives + ENROLLMENT_DIR=${TEMP_DIR}/enrollments + CHANNEL_MSP_DIR=${TEMP_DIR}/channel-msp + CONFIG_DIR=${TEMP_DIR}/fabric-rest-sample-config - mkdir -p build/fabric-rest-sample-config - - local peer_pem=build/msp/organizations/peerOrganizations/org1.example.com/msp/tlscacerts/org1-tls-ca.pem - local ca_pem=build/msp/organizations/peerOrganizations/org1.example.com/msp/cacerts/org1-ca.pem + mkdir -p $CONFIG_DIR + local peer_pem=$CHANNEL_MSP_DIR/peerOrganizations/org1/msp/tlscacerts/tlsca-signcert.pem + local ca_pem=$CHANNEL_MSP_DIR/peerOrganizations/org1/msp/cacerts/ca-signcert.pem echo "$(json_ccp 1 $peer_pem $ca_pem)" > build/fabric-rest-sample-config/HLF_CONNECTION_PROFILE_ORG1 - - peer_pem=build/msp/organizations/peerOrganizations/org2.example.com/msp/tlscacerts/org2-tls-ca.pem - ca_pem=build/msp/organizations/peerOrganizations/org2.example.com/msp/cacerts/org2-ca.pem + peer_pem=$CHANNEL_MSP_DIR/peerOrganizations/org2/msp/tlscacerts/tlsca-signcert.pem + ca_pem=$CHANNEL_MSP_DIR/peerOrganizations/org2/msp/cacerts/ca-signcert.pem echo "$(json_ccp 2 $peer_pem $ca_pem)" > build/fabric-rest-sample-config/HLF_CONNECTION_PROFILE_ORG2 - - cat build/msp/organizations/peerOrganizations/org1.example.com/users/Admin\@org1.example.com/msp/signcerts/cert.pem > build/fabric-rest-sample-config/HLF_CERTIFICATE_ORG1 - cat build/msp/organizations/peerOrganizations/org2.example.com/users/Admin\@org2.example.com/msp/signcerts/cert.pem > build/fabric-rest-sample-config/HLF_CERTIFICATE_ORG2 - cat build/msp/organizations/peerOrganizations/org1.example.com/users/Admin\@org1.example.com/msp/keystore/server.key > build/fabric-rest-sample-config/HLF_PRIVATE_KEY_ORG1 - cat build/msp/organizations/peerOrganizations/org2.example.com/users/Admin\@org2.example.com/msp/keystore/server.key > build/fabric-rest-sample-config/HLF_PRIVATE_KEY_ORG2 + cp $ENROLLMENT_DIR/org1/users/org1admin/msp/signcerts/cert.pem $CONFIG_DIR/HLF_CERTIFICATE_ORG1 + cp $ENROLLMENT_DIR/org2/users/org2admin/msp/signcerts/cert.pem $CONFIG_DIR/HLF_CERTIFICATE_ORG2 + + cp $ENROLLMENT_DIR/org1/users/org1admin/msp/keystore/key.pem $CONFIG_DIR/HLF_PRIVATE_KEY_ORG1 + cp $ENROLLMENT_DIR/org2/users/org2admin/msp/keystore/key.pem $CONFIG_DIR/HLF_PRIVATE_KEY_ORG2 kubectl -n $NS delete configmap fabric-rest-sample-config || true - kubectl -n $NS create configmap fabric-rest-sample-config --from-file=build/fabric-rest-sample-config/ + kubectl -n $NS create configmap fabric-rest-sample-config --from-file=$CONFIG_DIR pop_fn } -# todo: Make sure to port this to IKS / ICP -function ensure_rest_sample_image() { - push_fn "Ensuring fabric-rest-sample image" - - # todo: apply a tag / label to avoid pulling :latest from ghcr.io - - pop_fn 0 -} - function rollout_rest_sample() { push_fn "Starting fabric-rest-sample" @@ -77,15 +60,19 @@ function rollout_rest_sample() { } function launch_rest_sample() { - ensure_rest_sample_image + construct_rest_sample_configmap - rollout_rest_sample + + apply_template kube/fabric-rest-sample.yaml + + kubectl -n $NS rollout status deploy/fabric-rest-sample log "" - log "The fabric-rest-sample has started. See https://github.com/hyperledger/fabric-samples/tree/main/asset-transfer-basic/rest-api-typescript for additional usage." + log "The fabric-rest-sample has started." + log "See https://github.com/hyperledger/fabric-samples/tree/main/asset-transfer-basic/rest-api-typescript for additional usage details." log "To access the endpoint:" log "" log "export SAMPLE_APIKEY=97834158-3224-4CE7-95F9-A148C886653E" - log 'curl -s --header "X-Api-Key: ${SAMPLE_APIKEY}" http://localhost/api/assets' + log 'curl -s --header "X-Api-Key: ${SAMPLE_APIKEY}" http://fabric-rest-sample.'${DOMAIN}'/api/assets' log "" } \ No newline at end of file diff --git a/test-network-k8s/scripts/test_network.sh b/test-network-k8s/scripts/test_network.sh index d6e5fb79..c8965db8 100755 --- a/test-network-k8s/scripts/test_network.sh +++ b/test-network-k8s/scripts/test_network.sh @@ -5,20 +5,12 @@ # SPDX-License-Identifier: Apache-2.0 # -function launch() { - local yaml=$1 - cat ${yaml} \ - | sed 's,{{FABRIC_CONTAINER_REGISTRY}},'${FABRIC_CONTAINER_REGISTRY}',g' \ - | sed 's,{{FABRIC_VERSION}},'${FABRIC_VERSION}',g' \ - | kubectl -n $NS apply -f - -} - function launch_orderers() { push_fn "Launching orderers" - launch kube/org0/org0-orderer1.yaml - launch kube/org0/org0-orderer2.yaml - launch kube/org0/org0-orderer3.yaml + apply_template kube/org0/org0-orderer1.yaml + apply_template kube/org0/org0-orderer2.yaml + apply_template kube/org0/org0-orderer3.yaml kubectl -n $NS rollout status deploy/org0-orderer1 kubectl -n $NS rollout status deploy/org0-orderer2 @@ -30,10 +22,10 @@ function launch_orderers() { function launch_peers() { push_fn "Launching peers" - launch kube/org1/org1-peer1.yaml - launch kube/org1/org1-peer2.yaml - launch kube/org2/org2-peer1.yaml - launch kube/org2/org2-peer2.yaml + apply_template kube/org1/org1-peer1.yaml + apply_template kube/org1/org1-peer2.yaml + apply_template kube/org2/org2-peer1.yaml + apply_template kube/org2/org2-peer2.yaml kubectl -n $NS rollout status deploy/org1-peer1 kubectl -n $NS rollout status deploy/org1-peer2 @@ -43,155 +35,127 @@ function launch_peers() { pop_fn } -function create_org0_local_MSP() { - echo 'set -x +# todo: enroll org admin LOCALLY from the host OS +# fabric-ca-client register --id.name org1-admin --id.secret org1adminpw --id.type admin --url https://org1-ca --mspdir $FABRIC_CA_CLIENT_HOME/org1-ca/rcaadmin/msp --id.attrs "hf.Registrar.Roles=client,hf.Registrar.Attributes=*,hf.Revoker=true,hf.GenCRL=true,admin=true:ecert,abac.init=true:ecert" +# fabric-ca-client enroll --url https://org1-admin:org1adminpw@org1-ca --mspdir /var/hyperledger/fabric/organizations/peerOrganizations/org1.example.com/users/Admin@org1.example.com/msp +# cp /var/hyperledger/fabric/organizations/peerOrganizations/org1.example.com/users/Admin@org1.example.com/msp/keystore/*_sk /var/hyperledger/fabric/organizations/peerOrganizations/org1.example.com/users/Admin@org1.example.com/msp/keystore/server.key +# cp /var/hyperledger/fabric/organizations/peerOrganizations/org1.example.com/peers/org1-peer1.org1.example.com/msp/config.yaml /var/hyperledger/fabric/organizations/peerOrganizations/org1.example.com/users/Admin@org1.example.com/msp/config.yaml + +# fabric-ca-client register --id.name org1-admin --id.secret org1adminpw --id.type admin --url https://org1-ca --mspdir $FABRIC_CA_CLIENT_HOME/org1-ca/rcaadmin/msp --id.attrs "hf.Registrar.Roles=client,hf.Registrar.Attributes=*,hf.Revoker=true,hf.GenCRL=true,admin=true:ecert,abac.init=true:ecert" +# fabric-ca-client enroll --url https://org1-admin:org1adminpw@org1-ca --mspdir /var/hyperledger/fabric/organizations/peerOrganizations/org1.example.com/users/Admin@org1.example.com/msp +# cp /var/hyperledger/fabric/organizations/peerOrganizations/org1.example.com/users/Admin@org1.example.com/msp/keystore/*_sk /var/hyperledger/fabric/organizations/peerOrganizations/org1.example.com/users/Admin@org1.example.com/msp/keystore/server.key +# cp /var/hyperledger/fabric/organizations/peerOrganizations/org1.example.com/peers/org1-peer1.org1.example.com/msp/config.yaml /var/hyperledger/fabric/organizations/peerOrganizations/org1.example.com/users/Admin@org1.example.com/msp/config.yaml + +# fabric-ca-client register --id.name org2-admin --id.secret org2adminpw --id.type admin --url https://org2-ca --mspdir $FABRIC_CA_CLIENT_HOME/org2-ca/rcaadmin/msp --id.attrs "hf.Registrar.Roles=client,hf.Registrar.Attributes=*,hf.Revoker=true,hf.GenCRL=true,admin=true:ecert,abac.init=true:ecert" +# fabric-ca-client enroll --url https://org2-admin:org2adminpw@org2-ca --mspdir /var/hyperledger/fabric/organizations/peerOrganizations/org2.example.com/users/Admin@org2.example.com/msp +# cp /var/hyperledger/fabric/organizations/peerOrganizations/org2.example.com/users/Admin@org2.example.com/msp/keystore/*_sk /var/hyperledger/fabric/organizations/peerOrganizations/org2.example.com/users/Admin@org2.example.com/msp/keystore/server.key +# cp /var/hyperledger/fabric/organizations/peerOrganizations/org2.example.com/peers/org2-peer1.org2.example.com/msp/config.yaml /var/hyperledger/fabric/organizations/peerOrganizations/org2.example.com/users/Admin@org2.example.com/msp/config.yaml + + +# Each network node needs a registration, enrollment, and MSP config.yaml +function create_node_local_MSP() { + local node_type=$1 + local org=$2 + local node=$3 + local csr_hosts=$4 + local id_name=${org}-${node} + local id_secret=${node_type}pw + local ca_name=${org}-ca + + cat < /var/hyperledger/fabric/organizations/ordererOrganizations/org0.example.com/orderers/org0-orderer1.org0.example.com/msp/config.yaml - - cp /var/hyperledger/fabric/organizations/ordererOrganizations/org0.example.com/orderers/org0-orderer1.org0.example.com/msp/config.yaml /var/hyperledger/fabric/organizations/ordererOrganizations/org0.example.com/orderers/org0-orderer2.org0.example.com/msp/config.yaml - cp /var/hyperledger/fabric/organizations/ordererOrganizations/org0.example.com/orderers/org0-orderer1.org0.example.com/msp/config.yaml /var/hyperledger/fabric/organizations/ordererOrganizations/org0.example.com/orderers/org0-orderer3.org0.example.com/msp/config.yaml - ' | exec kubectl -n $NS exec deploy/org0-ca -i -- /bin/sh -} - -function create_org1_local_MSP() { - - echo 'set -x - export FABRIC_CA_CLIENT_HOME=/var/hyperledger/fabric-ca-client - export FABRIC_CA_CLIENT_TLS_CERTFILES=/var/hyperledger/fabric/config/tls/ca.crt - - # Each identity in the network needs a registration and enrollment. - fabric-ca-client register --id.name org1-peer1 --id.secret peerpw --id.type peer --url https://org1-ca --mspdir $FABRIC_CA_CLIENT_HOME/org1-ca/rcaadmin/msp - fabric-ca-client register --id.name org1-peer2 --id.secret peerpw --id.type peer --url https://org1-ca --mspdir $FABRIC_CA_CLIENT_HOME/org1-ca/rcaadmin/msp - fabric-ca-client register --id.name org1-admin --id.secret org1adminpw --id.type admin --url https://org1-ca --mspdir $FABRIC_CA_CLIENT_HOME/org1-ca/rcaadmin/msp --id.attrs "hf.Registrar.Roles=client,hf.Registrar.Attributes=*,hf.Revoker=true,hf.GenCRL=true,admin=true:ecert,abac.init=true:ecert" - - fabric-ca-client enroll --url https://org1-peer1:peerpw@org1-ca --csr.hosts localhost,org1-peer1,org1-peer-gateway-svc --mspdir /var/hyperledger/fabric/organizations/peerOrganizations/org1.example.com/peers/org1-peer1.org1.example.com/msp - fabric-ca-client enroll --url https://org1-peer2:peerpw@org1-ca --csr.hosts localhost,org1-peer2,org1-peer-gateway-svc --mspdir /var/hyperledger/fabric/organizations/peerOrganizations/org1.example.com/peers/org1-peer2.org1.example.com/msp - fabric-ca-client enroll --url https://org1-admin:org1adminpw@org1-ca --mspdir /var/hyperledger/fabric/organizations/peerOrganizations/org1.example.com/users/Admin@org1.example.com/msp - - cp /var/hyperledger/fabric/organizations/peerOrganizations/org1.example.com/users/Admin@org1.example.com/msp/keystore/*_sk /var/hyperledger/fabric/organizations/peerOrganizations/org1.example.com/users/Admin@org1.example.com/msp/keystore/server.key + fabric-ca-client enroll \ + --url https://${id_name}:${id_secret}@${ca_name} \ + --csr.hosts ${csr_hosts} \ + --mspdir /var/hyperledger/fabric/organizations/${node_type}Organizations/${org}.example.com/${node_type}s/${id_name}.${org}.example.com/msp # Create local MSP config.yaml echo "NodeOUs: Enable: true ClientOUIdentifier: - Certificate: cacerts/org1-ca.pem + Certificate: cacerts/${org}-ca.pem OrganizationalUnitIdentifier: client PeerOUIdentifier: - Certificate: cacerts/org1-ca.pem + Certificate: cacerts/${org}-ca.pem OrganizationalUnitIdentifier: peer AdminOUIdentifier: - Certificate: cacerts/org1-ca.pem + Certificate: cacerts/${org}-ca.pem OrganizationalUnitIdentifier: admin OrdererOUIdentifier: - Certificate: cacerts/org1-ca.pem - OrganizationalUnitIdentifier: orderer" > /var/hyperledger/fabric/organizations/peerOrganizations/org1.example.com/peers/org1-peer1.org1.example.com/msp/config.yaml - - - cp /var/hyperledger/fabric/organizations/peerOrganizations/org1.example.com/peers/org1-peer1.org1.example.com/msp/config.yaml /var/hyperledger/fabric/organizations/peerOrganizations/org1.example.com/peers/org1-peer2.org1.example.com/msp/config.yaml - cp /var/hyperledger/fabric/organizations/peerOrganizations/org1.example.com/peers/org1-peer1.org1.example.com/msp/config.yaml /var/hyperledger/fabric/organizations/peerOrganizations/org1.example.com/users/Admin@org1.example.com/msp/config.yaml - ' | exec kubectl -n $NS exec deploy/org1-ca -i -- /bin/sh - + Certificate: cacerts/${org}-ca.pem + OrganizationalUnitIdentifier: orderer" > /var/hyperledger/fabric/organizations/${node_type}Organizations/${org}.example.com/${node_type}s/${id_name}.${org}.example.com/msp/config.yaml +EOF } -function create_org2_local_MSP() { - echo 'set -x - export FABRIC_CA_CLIENT_HOME=/var/hyperledger/fabric-ca-client - export FABRIC_CA_CLIENT_TLS_CERTFILES=/var/hyperledger/fabric/config/tls/ca.crt +function create_orderer_local_MSP() { + local org=$1 + local orderer=$2 + local csr_hosts=${org}-${orderer} - # Each identity in the network needs a registration and enrollment. - fabric-ca-client register --id.name org2-peer1 --id.secret peerpw --id.type peer --url https://org2-ca --mspdir $FABRIC_CA_CLIENT_HOME/org2-ca/rcaadmin/msp - fabric-ca-client register --id.name org2-peer2 --id.secret peerpw --id.type peer --url https://org2-ca --mspdir $FABRIC_CA_CLIENT_HOME/org2-ca/rcaadmin/msp - fabric-ca-client register --id.name org2-admin --id.secret org2adminpw --id.type admin --url https://org2-ca --mspdir $FABRIC_CA_CLIENT_HOME/org2-ca/rcaadmin/msp --id.attrs "hf.Registrar.Roles=client,hf.Registrar.Attributes=*,hf.Revoker=true,hf.GenCRL=true,admin=true:ecert,abac.init=true:ecert" + create_node_local_MSP orderer $org $orderer $csr_hosts +} - fabric-ca-client enroll --url https://org2-peer1:peerpw@org2-ca --csr.hosts localhost,org2-peer1,org2-peer-gateway-svc --mspdir /var/hyperledger/fabric/organizations/peerOrganizations/org2.example.com/peers/org2-peer1.org2.example.com/msp - fabric-ca-client enroll --url https://org2-peer2:peerpw@org2-ca --csr.hosts localhost,org2-peer2,org2-peer-gateway-svc --mspdir /var/hyperledger/fabric/organizations/peerOrganizations/org2.example.com/peers/org2-peer2.org2.example.com/msp - fabric-ca-client enroll --url https://org2-admin:org2adminpw@org2-ca --mspdir /var/hyperledger/fabric/organizations/peerOrganizations/org2.example.com/users/Admin@org2.example.com/msp +function create_peer_local_MSP() { + local org=$1 + local peer=$2 + local csr_hosts=localhost,${org}-${peer},${org}-peer-gateway-svc - cp /var/hyperledger/fabric/organizations/peerOrganizations/org2.example.com/users/Admin@org2.example.com/msp/keystore/*_sk /var/hyperledger/fabric/organizations/peerOrganizations/org2.example.com/users/Admin@org2.example.com/msp/keystore/server.key - - # Create local MSP config.yaml - echo "NodeOUs: - Enable: true - ClientOUIdentifier: - Certificate: cacerts/org2-ca.pem - OrganizationalUnitIdentifier: client - PeerOUIdentifier: - Certificate: cacerts/org2-ca.pem - OrganizationalUnitIdentifier: peer - AdminOUIdentifier: - Certificate: cacerts/org2-ca.pem - OrganizationalUnitIdentifier: admin - OrdererOUIdentifier: - Certificate: cacerts/org2-ca.pem - OrganizationalUnitIdentifier: orderer" > /var/hyperledger/fabric/organizations/peerOrganizations/org2.example.com/peers/org2-peer1.org2.example.com/msp/config.yaml - - cp /var/hyperledger/fabric/organizations/peerOrganizations/org2.example.com/peers/org2-peer1.org2.example.com/msp/config.yaml /var/hyperledger/fabric/organizations/peerOrganizations/org2.example.com/peers/org2-peer2.org2.example.com/msp/config.yaml - cp /var/hyperledger/fabric/organizations/peerOrganizations/org2.example.com/peers/org2-peer1.org2.example.com/msp/config.yaml /var/hyperledger/fabric/organizations/peerOrganizations/org2.example.com/users/Admin@org2.example.com/msp/config.yaml - ' | exec kubectl -n $NS exec deploy/org2-ca -i -- /bin/sh + create_node_local_MSP peer $org $peer $csr_hosts } function create_local_MSP() { push_fn "Creating local node MSP" - create_org0_local_MSP - create_org1_local_MSP - create_org2_local_MSP - - pop_fn -} - -# TLS certificates are isused by the CA's Issuer, stored in a Kube secret, and mounted into the pod at /var/hyperledger/fabric/config/tls. -# For consistency with the Fabric-CA guide, his function copies the orderer's TLS certs into the traditional Fabric MSP / folder structure. -function extract_orderer_tls_cert() { - local orderer=$1 - - echo 'set -x - - mkdir -p /var/hyperledger/fabric/organizations/ordererOrganizations/org0.example.com/orderers/'${orderer}'.org0.example.com/tls/signcerts/ - - cp \ - var/hyperledger/fabric/config/tls/tls.crt \ - /var/hyperledger/fabric/organizations/ordererOrganizations/org0.example.com/orderers/'${orderer}'.org0.example.com/tls/signcerts/cert.pem - - ' | exec kubectl -n $NS exec deploy/${orderer} -i -c main -- /bin/sh -} - -function extract_orderer_tls_certs() { - push_fn "Extracting orderer TLS certs to local MSP folder" - - extract_orderer_tls_cert org0-orderer1 - extract_orderer_tls_cert org0-orderer2 - extract_orderer_tls_cert org0-orderer3 + create_orderer_local_MSP org0 orderer1 + create_orderer_local_MSP org0 orderer2 + create_orderer_local_MSP org0 orderer3 + + create_peer_local_MSP org1 peer1 + create_peer_local_MSP org1 peer2 + + create_peer_local_MSP org2 peer1 + create_peer_local_MSP org2 peer2 pop_fn } +# +## TLS certificates are isused by the CA's Issuer, stored in a Kube secret, and mounted into the pod at /var/hyperledger/fabric/config/tls. +## For consistency with the Fabric-CA guide, his function copies the orderer's TLS certs into the traditional Fabric MSP / folder structure. +#function extract_orderer_tls_cert() { +# local orderer=$1 +# +# echo 'set -x +# +# mkdir -p /var/hyperledger/fabric/organizations/ordererOrganizations/org0.example.com/orderers/'${orderer}'.org0.example.com/tls/signcerts/ +# +# cp \ +# var/hyperledger/fabric/config/tls/tls.crt \ +# /var/hyperledger/fabric/organizations/ordererOrganizations/org0.example.com/orderers/'${orderer}'.org0.example.com/tls/signcerts/cert.pem +# +# ' | exec kubectl -n $NS exec deploy/${orderer} -i -c main -- /bin/sh +#} +# +#function extract_orderer_tls_certs() { +# push_fn "Extracting orderer TLS certs to local MSP folder" +# +# extract_orderer_tls_cert org0-orderer1 +# extract_orderer_tls_cert org0-orderer2 +# extract_orderer_tls_cert org0-orderer3 +# +# pop_fn +#} function network_up() { @@ -213,17 +177,13 @@ function network_up() { launch_orderers launch_peers - extract_orderer_tls_certs +# extract_orderer_tls_certs } function stop_services() { push_fn "Stopping Fabric services" - # These pods are busy executing `sleep MAX_INT` and do not shut down very quickly... -# kubectl -n $NS delete deployment/org0-admin-cli --grace-period=0 --force -# kubectl -n $NS delete deployment/org1-admin-cli --grace-period=0 --force -# kubectl -n $NS delete deployment/org2-admin-cli --grace-period=0 --force - + kubectl -n $NS delete ingress --all kubectl -n $NS delete deployment --all kubectl -n $NS delete pod --all kubectl -n $NS delete service --all @@ -252,4 +212,6 @@ function scrub_org_volumes() { function network_down() { stop_services scrub_org_volumes + + rm -rf $PWD/build } diff --git a/test-network-k8s/scripts/utils.sh b/test-network-k8s/scripts/utils.sh index 14f4697b..0b97c07d 100644 --- a/test-network-k8s/scripts/utils.sh +++ b/test-network-k8s/scripts/utils.sh @@ -25,6 +25,8 @@ function logging_init() { function exit_fn() { rc=$? + set +x + # Write an error icon to the current logging statement. if [ "0" -ne $rc ]; then pop_fn $rc @@ -73,3 +75,29 @@ function pop_fn() { echo "" >> ${LOG_FILE} } +# Apply the current environment to a k8s template and apply to the cluster. +function apply_template() { + + echo "Applying template $1:" + cat $1 | envsubst + + cat $1 | envsubst | kubectl -n $NS apply -f - +} + +# Set the calling context to refer the peer binary to the correct org / peer instance +# +# todo: Expose the output of this function to a target that prints the context to STDOUT. +# +# e.g.: +# bash $ source $(network set-peer-context org1 peer2) +# bash $ peer chaincode list +# bash $ ... +function export_peer_context() { + local org=$1 + local peer=$2 + + export FABRIC_CFG_PATH=${PWD}/config/${org} + export CORE_PEER_ADDRESS=${org}-${peer}.${DOMAIN}:443 + export CORE_PEER_MSPCONFIGPATH=${TEMP_DIR}/enrollments/${org}/users/${org}admin/msp + export CORE_PEER_TLS_ROOTCERT_FILE=${TEMP_DIR}/channel-msp/peerOrganizations/${org}/msp/tlscacerts/tlsca-signcert.pem +}