From 05791d30bc68fee8f9408e34bc1d5cee68d842f9 Mon Sep 17 00:00:00 2001 From: "Mark S. Lewis" Date: Tue, 17 May 2022 12:49:22 +0100 Subject: [PATCH] Off-chain data sample using Fabric Gateway client API (#736) Signed-off-by: Mark S. Lewis --- README.md | 10 +- ci/scripts/run-test-network-basic.sh | 14 +- off_chain_data/README.md | 357 +++--------------- .../application-typescript/.eslintrc.yaml | 103 +++++ .../application-typescript/.gitignore | 17 + off_chain_data/application-typescript/.npmrc | 1 + .../application-typescript/package.json | 32 ++ .../application-typescript/src/app.ts | 57 +++ .../application-typescript/src/blockParser.ts | 160 ++++++++ .../application-typescript/src/connect.ts | 80 ++++ .../application-typescript/src/contract.ts | 54 +++ .../src/expectedError.ts | 12 + .../src/getAllAssets.ts | 27 ++ .../application-typescript/src/listen.ts | 257 +++++++++++++ .../application-typescript/src/transact.ts | 77 ++++ .../application-typescript/src/utils.ts | 75 ++++ .../application-typescript/tsconfig.json | 19 + .../.eslintrc.yaml | 10 + .../.gitignore | 0 .../legacy-application-javascript/README.md | 328 ++++++++++++++++ .../addAssets.js | 2 +- .../blockEventListener.js | 2 +- .../blockProcessing.js | 0 .../config.json | 0 .../couchdbutil.js | 0 .../deleteAsset.js | 2 +- .../enrollAdmin.js | 2 +- .../network-clean.sh | 0 .../package.json | 0 .../registerUser.js | 2 +- .../startFabric.sh | 4 +- .../transferAsset.js | 2 +- 32 files changed, 1397 insertions(+), 309 deletions(-) create mode 100644 off_chain_data/application-typescript/.eslintrc.yaml create mode 100644 off_chain_data/application-typescript/.gitignore create mode 100644 off_chain_data/application-typescript/.npmrc create mode 100644 off_chain_data/application-typescript/package.json create mode 100644 off_chain_data/application-typescript/src/app.ts create mode 100644 off_chain_data/application-typescript/src/blockParser.ts create mode 100644 off_chain_data/application-typescript/src/connect.ts create mode 100644 off_chain_data/application-typescript/src/contract.ts create mode 100644 off_chain_data/application-typescript/src/expectedError.ts create mode 100644 off_chain_data/application-typescript/src/getAllAssets.ts create mode 100644 off_chain_data/application-typescript/src/listen.ts create mode 100644 off_chain_data/application-typescript/src/transact.ts create mode 100644 off_chain_data/application-typescript/src/utils.ts create mode 100644 off_chain_data/application-typescript/tsconfig.json create mode 100644 off_chain_data/legacy-application-javascript/.eslintrc.yaml rename off_chain_data/{ => legacy-application-javascript}/.gitignore (100%) create mode 100644 off_chain_data/legacy-application-javascript/README.md rename off_chain_data/{ => legacy-application-javascript}/addAssets.js (96%) rename off_chain_data/{ => legacy-application-javascript}/blockEventListener.js (97%) rename off_chain_data/{ => legacy-application-javascript}/blockProcessing.js (100%) rename off_chain_data/{ => legacy-application-javascript}/config.json (100%) rename off_chain_data/{ => legacy-application-javascript}/couchdbutil.js (100%) rename off_chain_data/{ => legacy-application-javascript}/deleteAsset.js (92%) rename off_chain_data/{ => legacy-application-javascript}/enrollAdmin.js (92%) rename off_chain_data/{ => legacy-application-javascript}/network-clean.sh (100%) rename off_chain_data/{ => legacy-application-javascript}/package.json (100%) rename off_chain_data/{ => legacy-application-javascript}/registerUser.js (94%) rename off_chain_data/{ => legacy-application-javascript}/startFabric.sh (96%) rename off_chain_data/{ => legacy-application-javascript}/transferAsset.js (93%) diff --git a/README.md b/README.md index 1e51ef4a..cebf54fd 100644 --- a/README.md +++ b/README.md @@ -17,10 +17,10 @@ Organization peers and an ordering service node. You can use it on your local ma You can also use it to deploy and test your own Fabric chaincodes and applications. To get started, see the [test network tutorial](https://hyperledger-fabric.readthedocs.io/en/latest/test_network.html). -The [Kubernetes Test Network](test-network-k8s) sample builds upon the Compose network, constructing a Fabric -network with peer, orderer, and CA infrastructure nodes running on Kubernetes. In addition to providing a sample -Kubernetes guide, the Kube test network can be used as a platform to author and debug _cloud ready_ Fabric Client -applications on a development or CI workstation. +The [Kubernetes Test Network](test-network-k8s) sample builds upon the Compose network, constructing a Fabric +network with peer, orderer, and CA infrastructure nodes running on Kubernetes. In addition to providing a sample +Kubernetes guide, the Kube test network can be used as a platform to author and debug _cloud ready_ Fabric Client +applications on a development or CI workstation. @@ -51,7 +51,7 @@ Additional samples demonstrate various Fabric use cases and application patterns | **Sample** | **Description** | **Documentation** | | -------------|------------------------------|------------------| | [Commercial paper](commercial-paper) | Explore a use case and detailed application development tutorial in which two organizations use a blockchain network to trade commercial paper. | [Commercial paper tutorial](https://hyperledger-fabric.readthedocs.io/en/latest/tutorial/commercial_paper.html) | -| [Off chain data](off_chain_data) | Learn how to use the Peer channel-based event services to build an off-chain database for reporting and analytics. | [Peer channel-based event services](https://hyperledger-fabric.readthedocs.io/en/latest/peer_event_services.html) | +| [Off chain data](off_chain_data) | Learn how to use block events to build an off-chain database for reporting and analytics. | [Peer channel-based event services](https://hyperledger-fabric.readthedocs.io/en/latest/peer_event_services.html) | | [Token ERC-20](token-erc-20) | Smart contract demonstrating how to create and transfer fungible tokens using an account-based model. | [README](token-erc-20/README.md) | | [Token UTXO](token-utxo) | Smart contract demonstrating how to create and transfer fungible tokens using a UTXO (unspent transaction output) model. | [README](token-utxo/README.md) | | [Token ERC-1155](token-erc-1155) | Smart contract demonstrating how to create and transfer multiple tokens (both fungible and non-fungible) using an account based model. | [README](token-erc-1155/README.md) | diff --git a/ci/scripts/run-test-network-basic.sh b/ci/scripts/run-test-network-basic.sh index 7fe9185e..29620965 100755 --- a/ci/scripts/run-test-network-basic.sh +++ b/ci/scripts/run-test-network-basic.sh @@ -110,4 +110,16 @@ pushd ../asset-transfer-basic/application-gateway-go print "Executing AssetTransfer.go" go run . popd -stopNetwork \ No newline at end of file +stopNetwork + +# Run off-chain data TypeScript application +createNetwork +print "Initializing Typescript off-chain data application" +pushd ../off_chain_data/application-typescript +rm -f checkpoint.json store.log +npm install +print "Running the output app" +SIMULATED_FAILURE_COUNT=1 npm start getAllAssets transact getAllAssets listen +SIMULATED_FAILURE_COUNT=1 npm start listen +popd +stopNetwork diff --git a/off_chain_data/README.md b/off_chain_data/README.md index 79cde6da..22e64844 100644 --- a/off_chain_data/README.md +++ b/off_chain_data/README.md @@ -1,328 +1,95 @@ -# Off Chain data +# Off-chain data store sample -This sample demonstrates how you can use [Peer channel-based event services](https://hyperledger-fabric.readthedocs.io/en/latest/peer_event_services.html) -to replicate the data on your blockchain network to an off chain database. -Using an off chain database allows you to analyze the data from your network or -build a dashboard without degrading the performance of your application. +The off-chain data store sample demonstrates: -This sample uses the [Fabric network event listener](https://hyperledger.github.io/fabric-sdk-node/release-1.4/tutorial-channel-events.html) from the Node.JS Fabric SDK to write data to local instance of -CouchDB. +- Receiving block events in a client application. +- Using a checkpointer to resume event listening after a failure or application restart. +- Extracting ledger updates from block events in order to build an off-chain data store. -## Getting started +## About the sample -This sample uses Node Fabric SDK application code to connect to a running instance -of the Fabric test network. Make sure that you are running the following -commands from the `off_chain_data` directory. +This sample shows how to replicate the data in your blockchain network to an off-chain data store. Using an off-chain data store allows you to analyze the data from your network or build a dashboard without degrading the performance of your application. -### Starting the Network +This sample uses the block event listening capability of the [Fabric Gateway client API](https://hyperledger.github.io/fabric-gateway/) for Fabric v2.4 and later. -Use the following command to start the sample network: +### Application -``` -./startFabric.sh -``` +The client application provides several "commands" that can be invoked using the command-line: -This command will deploy an instance of the Fabric test network. The network -consists of an ordering service, two peer organizations with one peers each, and -a CA for each org. The command also creates a channel named `mychannel`. The -`asset-transfer-basic` chaincode will be installed on both peers and deployed to -the channel. +- **getAllAssets**: Retrieve the current details of all assets recorded on the ledger. See `application-typescript/src/getAllAssets.ts`. +- **listen**: Listen for block events, and use them to replicate ledger updates in an off-chain data store. See `application-typescript/src/listen.ts`. +- **transact**: Submit a set of transactions to create, modify and delete assets. See `application-typescript/src/transact.ts`. -### Configuration +To keep the sample code concise, the **listen** command writes ledger updates to an output file named `store.log` in the current working directory. A real implementation could write ledger updates directly to an off-chain data store of choice. You can inspect the information captured in this file as you run the sample. -The configuration for the listener is stored in the `config.json` file: +Note that the **listen** command is is restartable and will resume event listening after the last successfully processed block / transaction. This is achieved using a checkpointer to persist the current listening position. Checkpoint state is persisted to a file named `checkpoint.json` in the current working directory. If no checkpoint state is present, event listening begins from the start of the ledger (block number zero). -``` -{ - "peer_name": "peer0.org1.example.com", - "channelid": "mychannel", - "use_couchdb":true, - "create_history_log":true, - "couchdb_address": "http://admin:password@localhost:5990" -} -``` +### Smart Contract -`peer_name:` is the target peer for the listener. -`channelid:` is the channel name for block events. -`use_couchdb:` If set to true, events will be stored in a local instance of -CouchDB. If set to false, only a local log of events will be stored. -`create_history_log:` If true, a local log file will be created with all of the -block changes. -`couchdb_address:` is the local address for an off chain CouchDB database with username and password. +The asset-transfer-basic smart contract is used to generate transactions and associated ledger updates. -### Create an instance of CouchDB +## Running the sample -If you set the "use_couchdb" option to true in `config.json`, you can run the -following command start a local instance of CouchDB using docker: +The Fabric test network is used to deploy and run this sample. Follow these steps in order: -``` -docker run -e COUCHDB_USER=admin -e COUCHDB_PASSWORD=password --publish 5990:5984 --detach --name offchaindb couchdb:3.1.1 -docker start offchaindb -``` +1. Create the test network and a channel (from the `test-network` folder). + ``` + ./network.sh up createChannel -c mychannel -ca + ``` -### Install dependencies +1. Deploy one of the asset-transfer-basic smart contract implementations (from the `test-network` folder). -You need to install Node.js version 8.9.x to use the sample application code. -Execute the following commands to install the required dependencies: + ``` + # To deploy the TypeScript chaincode implementation + ./network.sh deployCC -ccn basic -ccp ../asset-transfer-basic/chaincode-typescript/ -ccl typescript -``` -npm install -``` + # To deploy the Go chaincode implementation + ./network.sh deployCC -ccn basic -ccp ../asset-transfer-basic/chaincode-go/ -ccl go -### Starting the Channel Event Listener + # To deploy the Java chaincode implementation + ./network.sh deployCC -ccn basic -ccp ../asset-transfer-basic/chaincode-java/ -ccl java + ``` -After we have installed the application dependencies, we can use the Node.js SDK -to create the identity our listener application will use to interact with the -network. Run the following command to enroll the admin user: -``` -node enrollAdmin.js -``` +1. Populate the ledger with some assets and use eventing to capture ledger updates (from the `off_chain_data` folder). -You can then run the following command to register and enroll an application -user: + ``` + # To run the TypeScript sample application + cd application-typescript + npm install + npm start transact listen + ``` -``` -node registerUser.js -``` +1. Interrupt the listener process using **Control-C**. -We can then use our application user to start the block event listener: +1. View the current world state of the blockchain (from the `off_chain_data` folder). You may want to compare the results to the ledger updates captured by the listener in the `store.log` file. -``` -node blockEventListener.js -``` + ``` + # To run the TypeScript sample application + cd application-typescript + npm --silent start getAllAssets + ``` -If the command is successful, you should see the output of the listener reading -the configuration blocks of `mychannel` in addition to the blocks that recorded -the approval and commitment of the assets chaincode definition. +1. Make some more ledger updates, then observe listener resume capability (from the `off_chain_data` folder). Note from the transaction IDs recorded to the console that the listener resumes from exactly after the last successfully processed transaction. -`blockEventListener.js` creates a listener named "offchain-listener" on the -channel `mychannel`. The listener writes each block added to the channel to a -processing map called BlockMap for temporary storage and ordering purposes. -`blockEventListener.js` uses `nextblock.txt` to keep track of the latest block -that was retrieved by the listener. The block number in `nextblock.txt` may be -set to a previous block number in order to replay previous blocks. The file -may also be deleted and all blocks will be replayed when the block listener is -started. + ``` + # To run the TypeScript sample application + cd application-typescript + npm start transact + SIMULATED_FAILURE_COUNT=5 npm start listen + npm start listen + ``` -`BlockProcessing.js` runs as a daemon and pulls each block in order from the -BlockMap. It then uses the read-write set of that block to extract the latest -key value data and store it in the database. The configuration blocks of -mychannel did not any data to the database because the blocks did not contain a -read-write set. - -The channel event listener also writes metadata from each block to a log file -defined as channelid_chaincodeid.log. In this example, events will be written to -a file named `mychannel_basic.log`. This allows you to record a history of -changes made by each block for each key in addition to storing the latest value -of the world state. - -**Note:** Leave the blockEventListener.js running in a terminal window. Open a -new window to execute the next parts of the demo. - -### Generate data on the blockchain - -Now that our listener is setup, we can generate data using the assets chaincode -and use our application to replicate the data to our database. Open a new -terminal and navigate to the `fabric-samples/off_chain_data` directory. - -You can use the `addAssets.js` file to add random sample data to blockchain. -The file uses the configuration information stored in `addAssets.json` to -create a series of assets. This file will be created during the first execution -of `addAssets.js` if it does not exist. This program can be run multiple times -without changing the properties. The `nextAssetNumber` will be incremented and -stored in the `addAssets.json` file. - -``` - { - "nextAssetNumber": 100, - "numberAssetsToAdd": 20 - } -``` - -Open a new window and run the following command to add 20 assets to the -blockchain: - -``` -node addAssets.js -``` - -After the assets have been added to the ledger, use the following command to -transfer one of the assets to a new owner: - -``` -node transferAsset.js asset110 james -``` - -Now run the following command to delete the asset that was transferred: - -``` -node deleteAsset.js asset110 -``` - -## Offchain CouchDB storage: - -If you followed the instructions above and set `use_couchdb` to true, -`blockEventListener.js` will create two tables in the local instance of CouchDB. -`blockEventListener.js` is written to create two tables for each channel and for -each chaincode. - -The first table is an offline representation of the current world state of the -blockchain ledger. This table was created using the read-write set data from -the blocks. If the listener is running, this table should be the same as the -latest values in the state database running on your peer. The table is named -after the channelid and chaincodeid, and is named mychannel_basic in this -example. You can navigate to this table using your browser: -http://127.0.0.1:5990/mychannel_basic/_all_docs - -A second table records each block as a historical record entry, and was created -using the block data that was recorded in the log file. The table name appends -history to the name of the first table, and is named mychannel_basic_history -in this example. You can also navigate to this table using your browser: -http://127.0.0.1:5990/mychannel_basic_history/_all_docs - -### Configure a map/reduce view for summarizing counts of assets by color: - -Now that we have state and history data replicated to tables in CouchDB, we -can use the following commands query our off-chain data. We will also add an -index to support a more complex query. Note that if the `blockEventListener.js` -is not running, the database commands below may fail since the database is only -created when events are received. - -Open a new terminal window and execute the following: - -``` -curl -X PUT http://127.0.0.1:5990/mychannel_basic/_design/colorviewdesign -d '{"views":{"colorview":{"map":"function (doc) { emit(doc.color, 1);}","reduce":"function ( keys , values , combine ) {return sum( values )}"}}}' -H 'Content-Type:application/json' -``` - -Execute a query to retrieve the total number of assets (reduce function): - -``` -curl -X GET http://127.0.0.1:5990/mychannel_basic/_design/colorviewdesign/_view/colorview?reduce=true -``` - -If successful, this command will return the number of assets in the blockchain -world state, without having to query the blockchain ledger: - -``` -{"rows":[ - {"key":null,"value":19} - ]} -``` - -Execute a new query to retrieve the number of assets by color (map function): - -``` -curl -X GET http://127.0.0.1:5990/mychannel_basic/_design/colorviewdesign/_view/colorview?group=true -``` - -The command will return a list of assets by color from the CouchDB database. - -``` -{"rows":[ - {"key":"blue","value":2}, - {"key":"green","value":2}, - {"key":"purple","value":3}, - {"key":"red","value":4}, - {"key":"white","value":6}, - {"key":"yellow","value":2} - ]} -``` - -To run a more complex command that reads through the block history database, we -will create an index of the blocknumber, sequence, and key fields. This index -will support a query that traces the history of each asset. Execute the -following command to create the index: - -``` -curl -X POST http://127.0.0.1:5990/mychannel_basic_history/_index -d '{"index":{"fields":["blocknumber", "sequence", "key"]},"name":"asset_history"}' -H 'Content-Type:application/json' -``` - -Now execute a query to retrieve the history for the asset we transferred and -then deleted: - -``` -curl -X POST http://127.0.0.1:5990/mychannel_basic_history/_find -d '{"selector":{"key":{"$eq":"asset110"}}, "fields":["blocknumber","is_delete","value"],"sort":[{"blocknumber":"asc"}, {"sequence":"asc"}]}' -H 'Content-Type:application/json' -``` - -You should see the transaction history of the asset that was created, -transferred, and then removed from the ledger. - -``` -{"docs":[ -{"blocknumber":12,"is_delete":false,"value":"{\"docType\":\"asset\",\"name\":\"asset110\",\"color\":\"blue\",\"size\":60,\"owner\":\"debra\"}"}, -{"blocknumber":22,"is_delete":false,"value":"{\"docType\":\"asset\",\"name\":\"asset110\",\"color\":\"blue\",\"size\":60,\"owner\":\"james\"}"}, -{"blocknumber":23,"is_delete":true,"value":""} - ]} -``` - -## Getting historical data from the network - -You can also use the `blockEventListener.js` program to retrieve historical data -from your network. This allows you to create a database that is up to date with -the latest data from the network or recover any blocks that the program may -have missed. - -If you ran through the example steps above, navigate back to the terminal window -where `blockEventListener.js` is running and close it. Once the listener is no -longer running, use the following command to add 20 more assets to the -ledger: - -``` -node addAssets.js -``` - -The listener will not be able to add the new assets to your CouchDB database. -If you check the current state table using the reduce command, you will only -be able to see the original assets in your database. - -``` -curl -X GET http://127.0.0.1:5990/mychannel_basic/_design/colorviewdesign/_view/colorview?reduce=true -``` - -To add the new data to your off-chain database, remove the `nextblock.txt` -file that kept track of the latest block read by `blockEventListener.js`: - -``` -rm nextblock.txt -``` - -You can new re-run the channel listener to read every block from the channel: - -``` -node blockEventListener.js -``` - -This will rebuild the CouchDB tables and include the 20 assets that have been -added to the ledger. If you run the reduce command against your database one -more time, - -``` -curl -X GET http://127.0.0.1:5990/mychannel_basic/_design/colorviewdesign/_view/colorview?reduce=true -``` - -you will be able to see that all of the assets have been added to your -database: - -``` -{"rows":[ -{"key":null,"value":39} -]} -``` +1. Interrupt the listener process using **Control-C**. ## Clean up -If you are finished using the sample application, you can bring down the network -and any accompanying artifacts by running the following command: -``` -./network-clean.sh -``` +The persisted event checkpoint position can be removed by deleting the `checkpoint.json` file while the listener is stopped. -Running the script will complete the following actions: +The recorded ledger updates can be removed by deleting the `store.log` file. -* Bring down the Fabric test network. -* Takes down the local CouchDB database. -* Remove the certificates you generated by deleting the `wallet` folder. -* Delete `nextblock.txt` so you can start with the first block next time you - operate the listener. -* Removes `addAssets.json`. +When you are finished, you can bring down the test network (from the `test-network` folder). The command will remove all the nodes of the test network, and delete any ledger data that you created. Be sure to remove the `checkpoint.json` and `store.log` files before attempting to run the application with a new network. + +``` +./network.sh down +``` \ No newline at end of file diff --git a/off_chain_data/application-typescript/.eslintrc.yaml b/off_chain_data/application-typescript/.eslintrc.yaml new file mode 100644 index 00000000..13c05fb5 --- /dev/null +++ b/off_chain_data/application-typescript/.eslintrc.yaml @@ -0,0 +1,103 @@ +env: + node: true + es2020: true +extends: + - eslint:recommended +rules: + arrow-spacing: + - error + comma-style: + - error + complexity: + - error + - 10 + eol-last: + - error + generator-star-spacing: + - error + - after + key-spacing: + - error + - beforeColon: false + afterColon: true + mode: minimum + keyword-spacing: + - error + no-multiple-empty-lines: + - error + no-trailing-spaces: + - error + no-whitespace-before-property: + - error + object-curly-newline: + - error + padded-blocks: + - error + - never + rest-spread-spacing: + - error + semi-style: + - error + space-before-blocks: + - error + space-in-parens: + - error + space-unary-ops: + - error + spaced-comment: + - error + template-curly-spacing: + - error + yield-star-spacing: + - error + - after +overrides: + - files: + - "**/*.ts" + parser: "@typescript-eslint/parser" + parserOptions: + sourceType: module + ecmaFeatures: + impliedStrict: true + project: "tsconfig.json" + tsconfigRootDir: "." + plugins: + - "@typescript-eslint" + extends: + - eslint:recommended + - plugin:@typescript-eslint/recommended + - plugin:@typescript-eslint/recommended-requiring-type-checking + rules: + "@typescript-eslint/comma-spacing": + - error + "@typescript-eslint/explicit-function-return-type": + - error + - allowExpressions: true + "@typescript-eslint/func-call-spacing": + - error + "@typescript-eslint/member-delimiter-style": + - error + "@typescript-eslint/indent": + - error + - 4 + - SwitchCase: 0 + "@typescript-eslint/prefer-nullish-coalescing": + - error + "@typescript-eslint/prefer-optional-chain": + - error + "@typescript-eslint/prefer-reduce-type-parameter": + - error + "@typescript-eslint/prefer-return-this-type": + - error + "@typescript-eslint/quotes": + - error + - single + "@typescript-eslint/type-annotation-spacing": + - error + "@typescript-eslint/semi": + - error + "@typescript-eslint/space-before-function-paren": + - error + - anonymous: never + named: never + asyncArrow: always diff --git a/off_chain_data/application-typescript/.gitignore b/off_chain_data/application-typescript/.gitignore new file mode 100644 index 00000000..28a349b2 --- /dev/null +++ b/off_chain_data/application-typescript/.gitignore @@ -0,0 +1,17 @@ +# +# SPDX-License-Identifier: Apache-2.0 +# + +# Coverage directory used by tools like istanbul +coverage + +# Dependency directories +node_modules/ +jspm_packages/ + +# Compiled TypeScript files +dist + +# Files generated by the application at runtime +checkpoint.json +store.log diff --git a/off_chain_data/application-typescript/.npmrc b/off_chain_data/application-typescript/.npmrc new file mode 100644 index 00000000..b6f27f13 --- /dev/null +++ b/off_chain_data/application-typescript/.npmrc @@ -0,0 +1 @@ +engine-strict=true diff --git a/off_chain_data/application-typescript/package.json b/off_chain_data/application-typescript/package.json new file mode 100644 index 00000000..a4e176be --- /dev/null +++ b/off_chain_data/application-typescript/package.json @@ -0,0 +1,32 @@ +{ + "name": "off-chain-data", + "version": "1.0.0", + "description": "Off-chain data store application", + "main": "dist/index.js", + "typings": "dist/index.d.ts", + "engines": { + "node": ">=14.17.0" + }, + "scripts": { + "build": "tsc", + "build:watch": "tsc -w", + "lint": "eslint ./src --ext .ts", + "prepare": "npm run build", + "pretest": "npm run lint", + "start": "node ./dist/app" + }, + "author": "Hyperledger", + "license": "Apache-2.0", + "devDependencies": { + "@tsconfig/node14": "^1.0.1", + "@types/node": "^14.18.16", + "@typescript-eslint/eslint-plugin": "^5.22.0", + "@typescript-eslint/parser": "^5.22.0", + "eslint": "^8.14.0", + "typescript": "~4.6.4" + }, + "dependencies": { + "@hyperledger/fabric-gateway": "^1.0.2-dev.20220505.18", + "@hyperledger/fabric-protos": "^0.1.0-dev.2300102001.1" + } +} diff --git a/off_chain_data/application-typescript/src/app.ts b/off_chain_data/application-typescript/src/app.ts new file mode 100644 index 00000000..eb9b3dc2 --- /dev/null +++ b/off_chain_data/application-typescript/src/app.ts @@ -0,0 +1,57 @@ +/* + * Copyright IBM Corp. All Rights Reserved. + * + * SPDX-License-Identifier: Apache-2.0 + */ + +import * as grpc from '@grpc/grpc-js'; +import { newGrpcConnection } from './connect'; +import { ExpectedError } from './expectedError'; +import { main as getAllAssets } from './getAllAssets'; +import { main as listen } from './listen'; +import { main as transact } from './transact'; + +const allCommands: Record Promise> = { + getAllAssets, + listen, + transact, +}; + +async function main(): Promise { + const commands = process.argv.slice(2).map(name => { + const command = allCommands[name]; + if (!command) { + printUsage(); + throw new Error(`Unknown command: ${name}`); + } + + return command; + }); + if (commands.length === 0) { + printUsage(); + throw new Error('Missing command'); + } + + const client = await newGrpcConnection(); + try { + for (const command of commands) { + await command(client); + } + } finally { + client.close(); + } +} + +function printUsage(): void { + console.log('Arguments: [ ...]'); + console.log('Available commands:', Object.keys(allCommands).join(', ')); +} + +main().catch(error => { + if (error instanceof ExpectedError) { + console.log(error); + } else { + console.error('\nUnexpected application error:', error); + process.exitCode = 1; + } +}); diff --git a/off_chain_data/application-typescript/src/blockParser.ts b/off_chain_data/application-typescript/src/blockParser.ts new file mode 100644 index 00000000..1bef9802 --- /dev/null +++ b/off_chain_data/application-typescript/src/blockParser.ts @@ -0,0 +1,160 @@ +/* + * Copyright IBM Corp. All Rights Reserved. + * + * SPDX-License-Identifier: Apache-2.0 + */ + +import { common, ledger, peer } from '@hyperledger/fabric-protos'; +import { assertDefined, cache } from './utils'; + +export interface Block { + getNumber(): bigint; + getTransactions(): Transaction[]; + toProto(): common.Block; +} + +export interface Transaction { + getChannelHeader(): common.ChannelHeader; + getValidationCode(): number; + isValid(): boolean; + getNamespaceReadWriteSets(): NamespaceReadWriteSet[]; + toProto(): common.Payload; +} + +export interface NamespaceReadWriteSet { + getNamespace(): string; + getReadWriteSet(): ledger.rwset.kvrwset.KVRWSet; + toProto(): ledger.rwset.NsReadWriteSet; +} + +export function parseBlock(block: common.Block): Block { + const validationCodes = getTransactionValidationCodes(block); + const header = assertDefined(block.getHeader(), 'Missing block header'); + + return { + getNumber: () => BigInt(header.getNumber()), + getTransactions: cache( + () => getPayloads(block) + .map((payload, i) => parsePayload(payload, validationCodes[i])) + .filter(payload => payload.isEndorserTransaction()) + .map(newTransaction) + ), + toProto: () => block, + }; +} + +interface Payload { + getChannelHeader(): common.ChannelHeader; + getEndorserTransaction(): EndorserTransaction; + getTransactionValidationCode(): number; + isEndorserTransaction(): boolean; + isValid(): boolean; + toProto(): common.Payload; +} + +interface EndorserTransaction { + getReadWriteSets(): ReadWriteSet[]; + toProto(): peer.Transaction; +} + +interface ReadWriteSet { + getNamespaceReadWriteSets(): NamespaceReadWriteSet[]; + toProto(): ledger.rwset.TxReadWriteSet; +} + +function parsePayload(payload: common.Payload, statusCode: number): Payload { + const cachedChannelHeader = cache(() => getChannelHeader(payload)); + const isEndorserTransaction = (): boolean => cachedChannelHeader().getType() === common.HeaderType.ENDORSER_TRANSACTION; + + return { + getChannelHeader: cachedChannelHeader, + getEndorserTransaction: cache( + () => { + if (!isEndorserTransaction()) { + throw new Error(`Unexpected payload type: ${cachedChannelHeader().getType()}`); + } + const transaction = peer.Transaction.deserializeBinary(payload.getData_asU8()); + return parseEndorserTransaction(transaction); + } + ), + getTransactionValidationCode: () => statusCode, + isEndorserTransaction, + isValid: () => statusCode === peer.TxValidationCode.VALID, + toProto: () => payload, + }; +} + +function parseEndorserTransaction(transaction: peer.Transaction): EndorserTransaction { + return { + getReadWriteSets: cache( + () => getChaincodeActionPayloads(transaction) + .map(payload => assertDefined(payload.getAction(), 'Missing chaincode endorsed action')) + .map(endorsedAction => endorsedAction.getProposalResponsePayload_asU8()) + .map(bytes => peer.ProposalResponsePayload.deserializeBinary(bytes)) + .map(responsePayload => peer.ChaincodeAction.deserializeBinary(responsePayload.getExtension_asU8())) + .map(chaincodeAction => chaincodeAction.getResults_asU8()) + .map(bytes => ledger.rwset.TxReadWriteSet.deserializeBinary(bytes)) + .map(parseReadWriteSet) + ), + toProto: () => transaction, + }; +} + +function newTransaction(payload: Payload): Transaction { + const transaction = payload.getEndorserTransaction(); + + return { + getChannelHeader: () => payload.getChannelHeader(), + getNamespaceReadWriteSets: () => transaction.getReadWriteSets() + .flatMap(readWriteSet => readWriteSet.getNamespaceReadWriteSets()), + getValidationCode: () => payload.getTransactionValidationCode(), + isValid: () => payload.isValid(), + toProto: () => payload.toProto(), + }; +} + +function parseReadWriteSet(readWriteSet: ledger.rwset.TxReadWriteSet): ReadWriteSet { + return { + getNamespaceReadWriteSets: () => { + if (readWriteSet.getDataModel() !== ledger.rwset.TxReadWriteSet.DataModel.KV) { + throw new Error(`Unexpected read/write set data model: ${readWriteSet.getDataModel()}`); + } + + return readWriteSet.getNsRwsetList().map(parseNamespaceReadWriteSet); + }, + toProto: () => readWriteSet, + }; +} + +function parseNamespaceReadWriteSet(nsReadWriteSet: ledger.rwset.NsReadWriteSet): NamespaceReadWriteSet { + return { + getNamespace: () => nsReadWriteSet.getNamespace(), + getReadWriteSet: cache( + () => ledger.rwset.kvrwset.KVRWSet.deserializeBinary(nsReadWriteSet.getRwset_asU8()) + ), + toProto: () => nsReadWriteSet, + }; +} + +function getTransactionValidationCodes(block: common.Block): Uint8Array { + const metadata = assertDefined(block.getMetadata(), 'Missing block metadata'); + return metadata.getMetadataList_asU8()[common.BlockMetadataIndex.TRANSACTIONS_FILTER]; +} + +function getPayloads(block: common.Block): common.Payload[] { + return (block.getData()?.getDataList_asU8() ?? []) + .map(bytes => common.Envelope.deserializeBinary(bytes)) + .map(envelope => envelope.getPayload_asU8()) + .map(bytes => common.Payload.deserializeBinary(bytes)); +} + +function getChannelHeader(payload: common.Payload): common.ChannelHeader { + const header = assertDefined(payload.getHeader(), 'Missing payload header'); + return common.ChannelHeader.deserializeBinary(header.getChannelHeader_asU8()); +} + +function getChaincodeActionPayloads(transaction: peer.Transaction): peer.ChaincodeActionPayload[] { + return transaction.getActionsList() + .map(transactionAction => transactionAction.getPayload_asU8()) + .map(bytes => peer.ChaincodeActionPayload.deserializeBinary(bytes)); +} diff --git a/off_chain_data/application-typescript/src/connect.ts b/off_chain_data/application-typescript/src/connect.ts new file mode 100644 index 00000000..b64fb049 --- /dev/null +++ b/off_chain_data/application-typescript/src/connect.ts @@ -0,0 +1,80 @@ +/* + * Copyright IBM Corp. All Rights Reserved. + * + * SPDX-License-Identifier: Apache-2.0 + */ + +import * as grpc from '@grpc/grpc-js'; +import { ConnectOptions, Identity, Signer, signers } from '@hyperledger/fabric-gateway'; +import * as crypto from 'crypto'; +import { promises as fs } from 'fs'; +import * as path from 'path'; + +export const channelName = process.env.CHANNEL_NAME ?? 'mychannel'; +export const chaincodeName = process.env.CHAINCODE_NAME ?? 'basic'; + +const peerName = 'peer0.org1.example.com'; +const mspId = process.env.MSP_ID ?? 'Org1MSP'; + +// Path to crypto materials. +const cryptoPath = path.resolve(process.env.CRYPTO_PATH ?? path.resolve(__dirname, '..', '..', '..', 'test-network', 'organizations', 'peerOrganizations', 'org1.example.com')); + +// Path to user private key directory. +const keyDirectoryPath = path.resolve(process.env.KEY_DIRECTORY_PATH ?? path.resolve(__dirname, cryptoPath, 'users', 'User1@org1.example.com', 'msp', 'keystore')); + +// Path to user certificate. +const certPath = path.resolve(process.env.CERT_PATH ?? path.resolve(__dirname, cryptoPath, 'users', 'User1@org1.example.com', 'msp', 'signcerts', 'cert.pem')); + +// Path to peer tls certificate. +const tlsCertPath = path.resolve(process.env.TLS_CERT_PATH ?? path.resolve(__dirname, cryptoPath, 'peers', peerName, 'tls', 'ca.crt')); + +// Gateway peer endpoint. +const peerEndpoint = process.env.PEER_ENDPOINT ?? 'localhost:7051'; + +// Gateway peer SSL host name override. +const peerHostAlias = process.env.PEER_HOST_ALIAS ?? peerName; + +export async function newGrpcConnection(): Promise { + const tlsRootCert = await fs.readFile(tlsCertPath); + const tlsCredentials = grpc.credentials.createSsl(tlsRootCert); + return new grpc.Client(peerEndpoint, tlsCredentials, { + 'grpc.ssl_target_name_override': peerHostAlias, + }); +} + +export async function newConnectOptions(client: grpc.Client): Promise { + return { + client, + identity: await newIdentity(), + signer: await newSigner(), + // Default timeouts for different gRPC calls + evaluateOptions: () => { + return { deadline: Date.now() + 5000 }; // 5 seconds + }, + endorseOptions: () => { + return { deadline: Date.now() + 15000 }; // 15 seconds + }, + submitOptions: () => { + return { deadline: Date.now() + 5000 }; // 5 seconds + }, + commitStatusOptions: () => { + return { deadline: Date.now() + 60000 }; // 1 minute + }, + }; +} + +async function newIdentity(): Promise { + const credentials = await fs.readFile(certPath); + return { mspId, credentials }; +} + +async function newSigner(): Promise { + const keyFiles = await fs.readdir(keyDirectoryPath); + if (keyFiles.length === 0) { + throw new Error(`No private key files found in directory ${keyDirectoryPath}`); + } + const keyPath = path.resolve(keyDirectoryPath, keyFiles[0]); + const privateKeyPem = await fs.readFile(keyPath); + const privateKey = crypto.createPrivateKey(privateKeyPem); + return signers.newPrivateKeySigner(privateKey); +} diff --git a/off_chain_data/application-typescript/src/contract.ts b/off_chain_data/application-typescript/src/contract.ts new file mode 100644 index 00000000..b14081d1 --- /dev/null +++ b/off_chain_data/application-typescript/src/contract.ts @@ -0,0 +1,54 @@ +/* + * Copyright IBM Corp. All Rights Reserved. + * + * SPDX-License-Identifier: Apache-2.0 + */ + +import { Contract } from '@hyperledger/fabric-gateway'; +import { TextDecoder } from 'util'; + +const utf8Decoder = new TextDecoder(); + +export interface Asset { + ID: string; + Color: string; + Size: number; + Owner: string; + AppriasedValue: number; +} + +export class AssetTransferBasic { + readonly #contract: Contract; + + constructor(contract: Contract) { + this.#contract = contract; + } + + async createAsset(asset: Asset): Promise { + await this.#contract.submit('CreateAsset', { + arguments: [asset.ID, asset.Color, String(asset.Size), asset.Owner, String(asset.AppriasedValue)], + }); + } + + async transferAsset(id: string, newOwner: string): Promise { + const result = await this.#contract.submit('TransferAsset', { + arguments: [id, newOwner], + }); + return utf8Decoder.decode(result); + } + + async deleteAsset(id: string): Promise { + await this.#contract.submit('DeleteAsset', { + arguments: [id], + }); + } + + async getAllAssets(): Promise { + const result = await this.#contract.evaluate('GetAllAssets'); + if (result.length === 0) { + return []; + } + + return JSON.parse(utf8Decoder.decode(result)) as Asset[]; + } +} diff --git a/off_chain_data/application-typescript/src/expectedError.ts b/off_chain_data/application-typescript/src/expectedError.ts new file mode 100644 index 00000000..296d87be --- /dev/null +++ b/off_chain_data/application-typescript/src/expectedError.ts @@ -0,0 +1,12 @@ +/* + * Copyright IBM Corp. All Rights Reserved. + * + * SPDX-License-Identifier: Apache-2.0 + */ + +export class ExpectedError extends Error { + constructor(message?: string) { + super(message); + this.name = ExpectedError.name; + } +} diff --git a/off_chain_data/application-typescript/src/getAllAssets.ts b/off_chain_data/application-typescript/src/getAllAssets.ts new file mode 100644 index 00000000..78f2d917 --- /dev/null +++ b/off_chain_data/application-typescript/src/getAllAssets.ts @@ -0,0 +1,27 @@ +/* + * Copyright IBM Corp. All Rights Reserved. + * + * SPDX-License-Identifier: Apache-2.0 + */ + +import { Client } from '@grpc/grpc-js'; +import { connect } from '@hyperledger/fabric-gateway'; +import { chaincodeName, channelName, newConnectOptions } from './connect'; +import { AssetTransferBasic } from './contract'; + +export async function main(client: Client): Promise { + const connectOptions = await newConnectOptions(client); + const gateway = connect(connectOptions); + + try { + const network = gateway.getNetwork(channelName); + const contract = network.getContract(chaincodeName); + + const smartContract = new AssetTransferBasic(contract); + const assets = await smartContract.getAllAssets(); + const assetsJson = JSON.stringify(assets, undefined, 2); + assetsJson.split('\n').forEach(line => console.log(line)); // Write line-by-line to avoid truncation + } finally { + gateway.close(); + } +} diff --git a/off_chain_data/application-typescript/src/listen.ts b/off_chain_data/application-typescript/src/listen.ts new file mode 100644 index 00000000..10dff94f --- /dev/null +++ b/off_chain_data/application-typescript/src/listen.ts @@ -0,0 +1,257 @@ +/* + * Copyright IBM Corp. All Rights Reserved. + * + * SPDX-License-Identifier: Apache-2.0 + */ + +import { Client } from '@grpc/grpc-js'; +import { Checkpointer, checkpointers, connect } from '@hyperledger/fabric-gateway'; +import { ledger } from '@hyperledger/fabric-protos'; +import { promises as fs } from 'fs'; +import * as path from 'path'; +import { TextDecoder } from 'util'; +import { Block, NamespaceReadWriteSet, parseBlock, Transaction } from './blockParser'; +import { channelName, newConnectOptions } from './connect'; +import { ExpectedError } from './expectedError'; + +const checkpointFile = path.resolve(process.env.CHECKPOINT_FILE ?? 'checkpoint.json'); +const storeFile = path.resolve(process.env.STORE_FILE ?? 'store.log'); +const simulatedFailureCount = getSimulatedFailureCount(); + +const startBlock = BigInt(0); +const utf8Decoder = new TextDecoder(); + +// Typically we should ignore read/write sets that apply to system chaincode namespaces. +const systemChaincodeNames = [ + '_lifecycle', + 'cscc', + 'escc', + 'lscc', + 'qscc', + 'vscc', +]; + +let transactionCount = 0; // Used only to simulate failures + +/** + * Apply writes for a given transaction to off-chain data store, ideally in a single operation for fault tolerance. + * @param data Transaction data. + */ +type StoreStrategy = (data: LedgerUpdate) => Promise; + +/** + * Ledger update made by a specific transaction. + */ +interface LedgerUpdate { + blockNumber: bigint; + channelName: string; + transactionId: string; + writes: Write[]; +} + +/** + * Description of a ledger write that can be applied to an off-chain data store. + */ +interface Write { + /** Channel whose ledger is being updated. */ + channelName: string; + /** Key name within the ledger namespace. */ + key: string; + /** Whether the key and associated value are being deleted. */ + isDelete: boolean; + /** Namespace within the ledger. */ + namespace: string; + /** If `isDelete` is false, the value written to the key; otherwise ignored. */ + value: Uint8Array; +} + +/** + * Apply writes for a given transaction to off-chain data store, ideally in a single operation for fault tolerance. + * This implementation just writes to a file. + */ +const applyWritesToOffChainStore: StoreStrategy = async (data) => { + simulateFailureIfRequired(); + + const writes = data.writes + .map(write => Object.assign({}, write, { + value: utf8Decoder.decode(write.value), // Convert bytes to text, purely for readability in output + })) + .map(write => JSON.stringify(write)); + + await fs.appendFile(storeFile, writes.join('\n') + '\n'); +}; + +export async function main(client: Client): Promise { + const connectOptions = await newConnectOptions(client); + const gateway = connect(connectOptions); + + try { + const network = gateway.getNetwork(channelName); + const checkpointer = await checkpointers.file(checkpointFile); + + console.log(`Starting event listening from block ${checkpointer.getBlockNumber() ?? startBlock}`); + console.log('Last processed transaction ID within block:', checkpointer.getTransactionId()); + if (simulatedFailureCount > 0) { + console.log(`Simulating a write failure every ${simulatedFailureCount} transactions`); + } + + const blocks = await network.getBlockEvents({ + checkpoint: checkpointer, + startBlock, // Used only if there is no checkpoint block number + }); + + try { + for await (const blockProto of blocks) { + const blockProcessor = new BlockProcessor({ + block: parseBlock(blockProto), + checkpointer, + store: applyWritesToOffChainStore, + }); + await blockProcessor.process(); + } + } finally { + blocks.close(); + } + } finally { + gateway.close(); + } +} + +interface BlockProcessorOptions { + block: Block; + checkpointer: Checkpointer; + store: StoreStrategy; +} + +class BlockProcessor { + readonly #block: Block; + readonly #checkpointer: Checkpointer; + readonly #store: StoreStrategy; + + constructor(options: Readonly) { + this.#block = options.block; + this.#checkpointer = options.checkpointer; + this.#store = options.store; + } + + async process(): Promise { + console.log(`\nReceived block ${this.#block.getNumber()}`); + + const blockNumber = this.#block.getNumber(); + const validTransactions = this.#getNewTransactions() + .filter(transaction => transaction.isValid()); + + for (const transaction of validTransactions) { + const transactionProcessor = new TransactionProcessor({ + blockNumber, + channelName: transaction.getChannelHeader().getChannelId(), + store: this.#store, + transaction, + }); + await transactionProcessor.process(); + + const transactionId = transaction.getChannelHeader().getTxId(); + await this.#checkpointer.checkpointTransaction(blockNumber, transactionId); + } + + await this.#checkpointer.checkpointBlock(this.#block.getNumber()); + } + + #getNewTransactions(): Transaction[] { + const transactions = this.#block.getTransactions(); + + const lastTransactionId = this.#checkpointer.getTransactionId(); + if (!lastTransactionId) { + // No previously processed transactions within this block so all are new + return transactions; + } + + // Ignore transactions up to the last processed transaction ID + const blockTransactionIds = transactions.map(transaction => transaction.getChannelHeader().getTxId()); + const lastProcessedIndex = blockTransactionIds.indexOf(lastTransactionId); + if (lastProcessedIndex < 0) { + throw new Error(`Checkpoint transaction ID ${lastTransactionId} not found in block ${this.#block.getNumber()} containing transactions: ${blockTransactionIds.join(', ')}`); + } + + return transactions.slice(lastProcessedIndex + 1); + } +} + +interface TransactionProcessorOptions { + blockNumber: bigint; + channelName: string; + store: StoreStrategy; + transaction: Transaction; +} + +class TransactionProcessor { + readonly #blockNumber: bigint; + readonly #channelName: string; + readonly #id: string; + readonly #namespaceReadWriteSets: NamespaceReadWriteSet[]; + readonly #store: StoreStrategy; + + constructor(options: Readonly) { + this.#blockNumber = options.blockNumber; + this.#channelName = options.channelName; + this.#id = options.transaction.getChannelHeader().getTxId(); + this.#namespaceReadWriteSets = options.transaction.getNamespaceReadWriteSets() + .filter(readWriteSet => !isSystemChaincode(readWriteSet.getNamespace())); + this.#store = options.store; + } + + async process(): Promise { + const writes = this.#getWrites(); + if (writes.length === 0) { + console.log(`Skipping read-only or system transaction ${this.#id}`); + return; + } + + console.log(`Process transaction ${this.#id}`); + + await this.#store({ + blockNumber: this.#blockNumber, + channelName: this.#channelName, + transactionId: this.#id, + writes: this.#getWrites(), + }); + } + + #getWrites(): Write[] { + return this.#namespaceReadWriteSets.flatMap(readWriteSet => { + const namespace = readWriteSet.getNamespace(); + return readWriteSet.getReadWriteSet().getWritesList().map(write => this.#newWrite(namespace, write)); + }); + } + + #newWrite(namespace: string, write: ledger.rwset.kvrwset.KVWrite): Write { + return { + channelName: this.#channelName, + key: write.getKey(), + isDelete: write.getIsDelete(), + namespace, + value: write.getValue_asU8(), + }; + } +} + +function isSystemChaincode(chaincodeName: string): boolean { + return systemChaincodeNames.includes(chaincodeName); +} + +function getSimulatedFailureCount(): number { + const value = process.env.SIMULATED_FAILURE_COUNT ?? '0'; + const count = Math.floor(Number(value)); + if (isNaN(count) || count < 0) { + throw new Error(`Invalid SIMULATED_FAILURE_COUNT value: ${String(value)}`); + } + + return count; +} + +function simulateFailureIfRequired(): void { + if (simulatedFailureCount > 0 && transactionCount++ >= simulatedFailureCount) { + transactionCount = 0; + throw new ExpectedError('Simulated write failure'); + } +} diff --git a/off_chain_data/application-typescript/src/transact.ts b/off_chain_data/application-typescript/src/transact.ts new file mode 100644 index 00000000..16c5a86d --- /dev/null +++ b/off_chain_data/application-typescript/src/transact.ts @@ -0,0 +1,77 @@ +/* + * Copyright IBM Corp. All Rights Reserved. + * + * SPDX-License-Identifier: Apache-2.0 + */ + +import { Client } from '@grpc/grpc-js'; +import { connect } from '@hyperledger/fabric-gateway'; +import * as crypto from 'crypto'; +import { chaincodeName, channelName, newConnectOptions } from './connect'; +import { Asset, AssetTransferBasic } from './contract'; +import { allFulfilled, differentElement, randomElement, randomInt } from './utils'; + +export async function main(client: Client): Promise { + const connectOptions = await newConnectOptions(client); + const gateway = connect(connectOptions); + + try { + const network = gateway.getNetwork(channelName); + const contract = network.getContract(chaincodeName); + + const smartContract = new AssetTransferBasic(contract); + const app = new TransactApp(smartContract); + await app.run(); + } finally { + gateway.close(); + } +} + +const colors = ['red', 'green', 'blue']; +const owners = ['alice', 'bob', 'charlie']; +const maxInitialValue = 1000; +const maxInitialSize = 10; + +class TransactApp { + readonly #smartContract: AssetTransferBasic; + #batchSize = 10; + + constructor(smartContract: AssetTransferBasic) { + this.#smartContract = smartContract; + } + + async run(): Promise { + const promises = Array.from({ length: this.#batchSize }, () => this.#transact()); + await allFulfilled(promises); + } + + async #transact(): Promise { + const asset = this.#newAsset(); + + await this.#smartContract.createAsset(asset); + console.log(`Created asset ${asset.ID}`); + + // Transfer randomly 1 in 2 assets to a new owner. + if (randomInt(2) === 0) { + const newOwner = differentElement(owners, asset.Owner); + const oldOwner = await this.#smartContract.transferAsset(asset.ID, newOwner); + console.log(`Transferred asset ${asset.ID} from ${oldOwner} to ${newOwner}`); + } + + // Delete randomly 1 in 4 created assets. + if (randomInt(4) === 0) { + await this.#smartContract.deleteAsset(asset.ID); + console.log(`Deleted asset ${asset.ID}`); + } + } + + #newAsset(): Asset { + return { + ID: crypto.randomUUID(), + Color: randomElement(colors), + Size: randomInt(maxInitialSize) + 1, + Owner: randomElement(owners), + AppriasedValue: randomInt(maxInitialValue) + 1, + }; + } +} diff --git a/off_chain_data/application-typescript/src/utils.ts b/off_chain_data/application-typescript/src/utils.ts new file mode 100644 index 00000000..b58ca371 --- /dev/null +++ b/off_chain_data/application-typescript/src/utils.ts @@ -0,0 +1,75 @@ +/* + * Copyright IBM Corp. All Rights Reserved. + * + * SPDX-License-Identifier: Apache-2.0 + */ + +/** + * Pick a random element from an array. + * @param values Candidate elements. + */ +export function randomElement(values: T[]): T { + return values[randomInt(values.length)]; +} + +/** + * Generate a random integer in the range 0 to max - 1. + * @param max Maximum value (exclusive). + */ +export function randomInt(max: number): number { + return Math.floor(Math.random() * max); +} + +/** + * Pick a random element from an array, excluding the current value. + * @param values Candidate elements. + * @param currentValue Value to avoid. + */ +export function differentElement(values: T[], currentValue: T): T { + const candidateValues = values.filter(value => value !== currentValue); + return randomElement(candidateValues); +} + +/** + * Wait for all promises to complete, then throw an Error only if any of the promises were rejected. + * @param promises Promises to be awaited. + */ +export async function allFulfilled(promises: Promise[]): Promise { + const results = await Promise.allSettled(promises); + const failures = results + .map(result => result.status === 'rejected' && result.reason as unknown) + .filter(reason => !!reason); + + if (failures.length > 0) { + const failMessages = ' - ' + failures.join('\n - '); + throw new Error(`${failures.length} failures:\n${failMessages}\n`); + } +} + +/** + * Return the value if it is defined; otherwise thrown an error. + * @param value A value that might not be defined. + * @param message Error message if the value is not defined. + */ +export function assertDefined(value: T | null | undefined, message: string): T { + if (value == undefined) { + throw new Error(message); + } + + return value; +} + +/** + * Wrap a function call with a cache. On first call the wrapped function is invoked to obtain a result. Subsequent + * calls return the cached result. + * @param f A function whose result should be cached. + */ +export function cache(f: () => T): () => T { + let value: T | undefined; + return () => { + if (value === undefined) { + value = f(); + } + return value; + }; +} diff --git a/off_chain_data/application-typescript/tsconfig.json b/off_chain_data/application-typescript/tsconfig.json new file mode 100644 index 00000000..9436c6bb --- /dev/null +++ b/off_chain_data/application-typescript/tsconfig.json @@ -0,0 +1,19 @@ +{ + "$schema": "https://json.schemastore.org/tsconfig", + "extends": "@tsconfig/node14/tsconfig.json", + "compilerOptions": { + "declaration": true, + "declarationMap": true, + "sourceMap": true, + "outDir": "dist", + "rootDir": "src", + "noUnusedLocals": true, + "noImplicitReturns": true + }, + "include": [ + "src/" + ], + "exclude": [ + "src/**/*.spec.ts" + ] +} diff --git a/off_chain_data/legacy-application-javascript/.eslintrc.yaml b/off_chain_data/legacy-application-javascript/.eslintrc.yaml new file mode 100644 index 00000000..050ac5cc --- /dev/null +++ b/off_chain_data/legacy-application-javascript/.eslintrc.yaml @@ -0,0 +1,10 @@ +env: + browser: true + commonjs: true + es6: true +globals: + Atomics: readonly + SharedArrayBuffer: readonly +parserOptions: + ecmaVersion: 2018 +rules: {} diff --git a/off_chain_data/.gitignore b/off_chain_data/legacy-application-javascript/.gitignore similarity index 100% rename from off_chain_data/.gitignore rename to off_chain_data/legacy-application-javascript/.gitignore diff --git a/off_chain_data/legacy-application-javascript/README.md b/off_chain_data/legacy-application-javascript/README.md new file mode 100644 index 00000000..fd92045c --- /dev/null +++ b/off_chain_data/legacy-application-javascript/README.md @@ -0,0 +1,328 @@ +# Off Chain data + +This sample demonstrates how you can use [Peer channel-based event services](https://hyperledger-fabric.readthedocs.io/en/latest/peer_event_services.html) +to replicate the data on your blockchain network to an off chain database. +Using an off chain database allows you to analyze the data from your network or +build a dashboard without degrading the performance of your application. + +This sample uses the [Fabric network event listener](https://hyperledger.github.io/fabric-sdk-node/release-1.4/tutorial-channel-events.html) from the Node.JS Fabric SDK to write data to local instance of +CouchDB. + +## Getting started + +This sample uses Node Fabric SDK application code to connect to a running instance +of the Fabric test network. Make sure that you are running the following +commands from the `off_chain_data/legacy-javascript` directory. + +### Starting the Network + +Use the following command to start the sample network: + +``` +./startFabric.sh +``` + +This command will deploy an instance of the Fabric test network. The network +consists of an ordering service, two peer organizations with one peers each, and +a CA for each org. The command also creates a channel named `mychannel`. The +`asset-transfer-basic` chaincode will be installed on both peers and deployed to +the channel. + +### Configuration + +The configuration for the listener is stored in the `config.json` file: + +``` +{ + "peer_name": "peer0.org1.example.com", + "channelid": "mychannel", + "use_couchdb":true, + "create_history_log":true, + "couchdb_address": "http://admin:password@localhost:5990" +} +``` + +`peer_name:` is the target peer for the listener. +`channelid:` is the channel name for block events. +`use_couchdb:` If set to true, events will be stored in a local instance of +CouchDB. If set to false, only a local log of events will be stored. +`create_history_log:` If true, a local log file will be created with all of the +block changes. +`couchdb_address:` is the local address for an off chain CouchDB database with username and password. + +### Create an instance of CouchDB + +If you set the "use_couchdb" option to true in `config.json`, you can run the +following command start a local instance of CouchDB using docker: + +``` +docker run -e COUCHDB_USER=admin -e COUCHDB_PASSWORD=password --publish 5990:5984 --detach --name offchaindb couchdb:3.1.1 +docker start offchaindb +``` + + +### Install dependencies + +You need to install Node.js version 8.9.x to use the sample application code. +Execute the following commands to install the required dependencies: + +``` +npm install +``` + +### Starting the Channel Event Listener + +After we have installed the application dependencies, we can use the Node.js SDK +to create the identity our listener application will use to interact with the +network. Run the following command to enroll the admin user: +``` +node enrollAdmin.js +``` + +You can then run the following command to register and enroll an application +user: + +``` +node registerUser.js +``` + +We can then use our application user to start the block event listener: + +``` +node blockEventListener.js +``` + +If the command is successful, you should see the output of the listener reading +the configuration blocks of `mychannel` in addition to the blocks that recorded +the approval and commitment of the assets chaincode definition. + +`blockEventListener.js` creates a listener named "offchain-listener" on the +channel `mychannel`. The listener writes each block added to the channel to a +processing map called BlockMap for temporary storage and ordering purposes. +`blockEventListener.js` uses `nextblock.txt` to keep track of the latest block +that was retrieved by the listener. The block number in `nextblock.txt` may be +set to a previous block number in order to replay previous blocks. The file +may also be deleted and all blocks will be replayed when the block listener is +started. + +`BlockProcessing.js` runs as a daemon and pulls each block in order from the +BlockMap. It then uses the read-write set of that block to extract the latest +key value data and store it in the database. The configuration blocks of +mychannel did not any data to the database because the blocks did not contain a +read-write set. + +The channel event listener also writes metadata from each block to a log file +defined as channelid_chaincodeid.log. In this example, events will be written to +a file named `mychannel_basic.log`. This allows you to record a history of +changes made by each block for each key in addition to storing the latest value +of the world state. + +**Note:** Leave the blockEventListener.js running in a terminal window. Open a +new window to execute the next parts of the demo. + +### Generate data on the blockchain + +Now that our listener is setup, we can generate data using the assets chaincode +and use our application to replicate the data to our database. Open a new +terminal and navigate to the `fabric-samples/off_chain_data/legacy-javascript` directory. + +You can use the `addAssets.js` file to add random sample data to blockchain. +The file uses the configuration information stored in `addAssets.json` to +create a series of assets. This file will be created during the first execution +of `addAssets.js` if it does not exist. This program can be run multiple times +without changing the properties. The `nextAssetNumber` will be incremented and +stored in the `addAssets.json` file. + +``` + { + "nextAssetNumber": 100, + "numberAssetsToAdd": 20 + } +``` + +Open a new window and run the following command to add 20 assets to the +blockchain: + +``` +node addAssets.js +``` + +After the assets have been added to the ledger, use the following command to +transfer one of the assets to a new owner: + +``` +node transferAsset.js asset110 james +``` + +Now run the following command to delete the asset that was transferred: + +``` +node deleteAsset.js asset110 +``` + +## Offchain CouchDB storage: + +If you followed the instructions above and set `use_couchdb` to true, +`blockEventListener.js` will create two tables in the local instance of CouchDB. +`blockEventListener.js` is written to create two tables for each channel and for +each chaincode. + +The first table is an offline representation of the current world state of the +blockchain ledger. This table was created using the read-write set data from +the blocks. If the listener is running, this table should be the same as the +latest values in the state database running on your peer. The table is named +after the channelid and chaincodeid, and is named mychannel_basic in this +example. You can navigate to this table using your browser: +http://127.0.0.1:5990/mychannel_basic/_all_docs + +A second table records each block as a historical record entry, and was created +using the block data that was recorded in the log file. The table name appends +history to the name of the first table, and is named mychannel_basic_history +in this example. You can also navigate to this table using your browser: +http://127.0.0.1:5990/mychannel_basic_history/_all_docs + +### Configure a map/reduce view for summarizing counts of assets by color: + +Now that we have state and history data replicated to tables in CouchDB, we +can use the following commands query our off-chain data. We will also add an +index to support a more complex query. Note that if the `blockEventListener.js` +is not running, the database commands below may fail since the database is only +created when events are received. + +Open a new terminal window and execute the following: + +``` +curl -X PUT http://127.0.0.1:5990/mychannel_basic/_design/colorviewdesign -d '{"views":{"colorview":{"map":"function (doc) { emit(doc.color, 1);}","reduce":"function ( keys , values , combine ) {return sum( values )}"}}}' -H 'Content-Type:application/json' +``` + +Execute a query to retrieve the total number of assets (reduce function): + +``` +curl -X GET http://127.0.0.1:5990/mychannel_basic/_design/colorviewdesign/_view/colorview?reduce=true +``` + +If successful, this command will return the number of assets in the blockchain +world state, without having to query the blockchain ledger: + +``` +{"rows":[ + {"key":null,"value":19} + ]} +``` + +Execute a new query to retrieve the number of assets by color (map function): + +``` +curl -X GET http://127.0.0.1:5990/mychannel_basic/_design/colorviewdesign/_view/colorview?group=true +``` + +The command will return a list of assets by color from the CouchDB database. + +``` +{"rows":[ + {"key":"blue","value":2}, + {"key":"green","value":2}, + {"key":"purple","value":3}, + {"key":"red","value":4}, + {"key":"white","value":6}, + {"key":"yellow","value":2} + ]} +``` + +To run a more complex command that reads through the block history database, we +will create an index of the blocknumber, sequence, and key fields. This index +will support a query that traces the history of each asset. Execute the +following command to create the index: + +``` +curl -X POST http://127.0.0.1:5990/mychannel_basic_history/_index -d '{"index":{"fields":["blocknumber", "sequence", "key"]},"name":"asset_history"}' -H 'Content-Type:application/json' +``` + +Now execute a query to retrieve the history for the asset we transferred and +then deleted: + +``` +curl -X POST http://127.0.0.1:5990/mychannel_basic_history/_find -d '{"selector":{"key":{"$eq":"asset110"}}, "fields":["blocknumber","is_delete","value"],"sort":[{"blocknumber":"asc"}, {"sequence":"asc"}]}' -H 'Content-Type:application/json' +``` + +You should see the transaction history of the asset that was created, +transferred, and then removed from the ledger. + +``` +{"docs":[ +{"blocknumber":12,"is_delete":false,"value":"{\"docType\":\"asset\",\"name\":\"asset110\",\"color\":\"blue\",\"size\":60,\"owner\":\"debra\"}"}, +{"blocknumber":22,"is_delete":false,"value":"{\"docType\":\"asset\",\"name\":\"asset110\",\"color\":\"blue\",\"size\":60,\"owner\":\"james\"}"}, +{"blocknumber":23,"is_delete":true,"value":""} + ]} +``` + +## Getting historical data from the network + +You can also use the `blockEventListener.js` program to retrieve historical data +from your network. This allows you to create a database that is up to date with +the latest data from the network or recover any blocks that the program may +have missed. + +If you ran through the example steps above, navigate back to the terminal window +where `blockEventListener.js` is running and close it. Once the listener is no +longer running, use the following command to add 20 more assets to the +ledger: + +``` +node addAssets.js +``` + +The listener will not be able to add the new assets to your CouchDB database. +If you check the current state table using the reduce command, you will only +be able to see the original assets in your database. + +``` +curl -X GET http://127.0.0.1:5990/mychannel_basic/_design/colorviewdesign/_view/colorview?reduce=true +``` + +To add the new data to your off-chain database, remove the `nextblock.txt` +file that kept track of the latest block read by `blockEventListener.js`: + +``` +rm nextblock.txt +``` + +You can new re-run the channel listener to read every block from the channel: + +``` +node blockEventListener.js +``` + +This will rebuild the CouchDB tables and include the 20 assets that have been +added to the ledger. If you run the reduce command against your database one +more time, + +``` +curl -X GET http://127.0.0.1:5990/mychannel_basic/_design/colorviewdesign/_view/colorview?reduce=true +``` + +you will be able to see that all of the assets have been added to your +database: + +``` +{"rows":[ +{"key":null,"value":39} +]} +``` + +## Clean up + +If you are finished using the sample application, you can bring down the network +and any accompanying artifacts by running the following command: +``` +./network-clean.sh +``` + +Running the script will complete the following actions: + +* Bring down the Fabric test network. +* Takes down the local CouchDB database. +* Remove the certificates you generated by deleting the `wallet` folder. +* Delete `nextblock.txt` so you can start with the first block next time you + operate the listener. +* Removes `addAssets.json`. diff --git a/off_chain_data/addAssets.js b/off_chain_data/legacy-application-javascript/addAssets.js similarity index 96% rename from off_chain_data/addAssets.js rename to off_chain_data/legacy-application-javascript/addAssets.js index 1137d5f7..38119571 100644 --- a/off_chain_data/addAssets.js +++ b/off_chain_data/legacy-application-javascript/addAssets.js @@ -69,7 +69,7 @@ async function main() { } // Parse the connection profile. - const ccpPath = path.resolve(__dirname, '..', 'test-network','organizations','peerOrganizations','org1.example.com', 'connection-org1.json'); + const ccpPath = path.resolve(__dirname, '..', '..', 'test-network','organizations','peerOrganizations','org1.example.com', 'connection-org1.json'); const ccp = JSON.parse(fs.readFileSync(ccpPath, 'utf8')); // Configure a wallet. This wallet must already be primed with an identity that diff --git a/off_chain_data/blockEventListener.js b/off_chain_data/legacy-application-javascript/blockEventListener.js similarity index 97% rename from off_chain_data/blockEventListener.js rename to off_chain_data/legacy-application-javascript/blockEventListener.js index 43749c3b..610ded88 100644 --- a/off_chain_data/blockEventListener.js +++ b/off_chain_data/legacy-application-javascript/blockEventListener.js @@ -107,7 +107,7 @@ async function main() { // Parse the connection profile. This would be the path to the file downloaded // from the IBM Blockchain Platform operational console. - const ccpPath = path.resolve(__dirname, '..', 'test-network','organizations','peerOrganizations','org1.example.com', 'connection-org1.json'); + const ccpPath = path.resolve(__dirname, '..', '..', 'test-network','organizations','peerOrganizations','org1.example.com', 'connection-org1.json'); const ccp = JSON.parse(fs.readFileSync(ccpPath, 'utf8')); // Create a new gateway for connecting to our peer node. const gateway = new Gateway(); diff --git a/off_chain_data/blockProcessing.js b/off_chain_data/legacy-application-javascript/blockProcessing.js similarity index 100% rename from off_chain_data/blockProcessing.js rename to off_chain_data/legacy-application-javascript/blockProcessing.js diff --git a/off_chain_data/config.json b/off_chain_data/legacy-application-javascript/config.json similarity index 100% rename from off_chain_data/config.json rename to off_chain_data/legacy-application-javascript/config.json diff --git a/off_chain_data/couchdbutil.js b/off_chain_data/legacy-application-javascript/couchdbutil.js similarity index 100% rename from off_chain_data/couchdbutil.js rename to off_chain_data/legacy-application-javascript/couchdbutil.js diff --git a/off_chain_data/deleteAsset.js b/off_chain_data/legacy-application-javascript/deleteAsset.js similarity index 92% rename from off_chain_data/deleteAsset.js rename to off_chain_data/legacy-application-javascript/deleteAsset.js index aeede815..37a8ff58 100644 --- a/off_chain_data/deleteAsset.js +++ b/off_chain_data/legacy-application-javascript/deleteAsset.js @@ -35,7 +35,7 @@ async function main() { try { // Parse the connection profile. - const ccpPath = path.resolve(__dirname, '..', 'test-network','organizations','peerOrganizations','org1.example.com', 'connection-org1.json'); + const ccpPath = path.resolve(__dirname, '..', '..', 'test-network','organizations','peerOrganizations','org1.example.com', 'connection-org1.json'); const ccp = JSON.parse(fs.readFileSync(ccpPath, 'utf8')); // Configure a wallet. This wallet must already be primed with an identity that diff --git a/off_chain_data/enrollAdmin.js b/off_chain_data/legacy-application-javascript/enrollAdmin.js similarity index 92% rename from off_chain_data/enrollAdmin.js rename to off_chain_data/legacy-application-javascript/enrollAdmin.js index 9cc7c996..fe4577e6 100644 --- a/off_chain_data/enrollAdmin.js +++ b/off_chain_data/legacy-application-javascript/enrollAdmin.js @@ -14,7 +14,7 @@ const path = require('path'); async function main() { try { // load the network configuration - const ccpPath = path.resolve(__dirname, '..', 'test-network', 'organizations', 'peerOrganizations', 'org1.example.com', 'connection-org1.json'); + const ccpPath = path.resolve(__dirname, '..', '..', 'test-network', 'organizations', 'peerOrganizations', 'org1.example.com', 'connection-org1.json'); let ccp = JSON.parse(fs.readFileSync(ccpPath, 'utf8')); // Create a new CA client for interacting with the CA. diff --git a/off_chain_data/network-clean.sh b/off_chain_data/legacy-application-javascript/network-clean.sh similarity index 100% rename from off_chain_data/network-clean.sh rename to off_chain_data/legacy-application-javascript/network-clean.sh diff --git a/off_chain_data/package.json b/off_chain_data/legacy-application-javascript/package.json similarity index 100% rename from off_chain_data/package.json rename to off_chain_data/legacy-application-javascript/package.json diff --git a/off_chain_data/registerUser.js b/off_chain_data/legacy-application-javascript/registerUser.js similarity index 94% rename from off_chain_data/registerUser.js rename to off_chain_data/legacy-application-javascript/registerUser.js index 1e004e1c..c1cefc5b 100644 --- a/off_chain_data/registerUser.js +++ b/off_chain_data/legacy-application-javascript/registerUser.js @@ -14,7 +14,7 @@ const path = require('path'); async function main() { try { // load the network configuration - const ccpPath = path.resolve(__dirname, '..', 'test-network','organizations','peerOrganizations','org1.example.com', 'connection-org1.json'); + const ccpPath = path.resolve(__dirname, '..', '..', 'test-network','organizations','peerOrganizations','org1.example.com', 'connection-org1.json'); const ccp = JSON.parse(fs.readFileSync(ccpPath, 'utf8')); // Create a new CA client for interacting with the CA. diff --git a/off_chain_data/startFabric.sh b/off_chain_data/legacy-application-javascript/startFabric.sh similarity index 96% rename from off_chain_data/startFabric.sh rename to off_chain_data/legacy-application-javascript/startFabric.sh index b22eefc8..3dc1a97a 100755 --- a/off_chain_data/startFabric.sh +++ b/off_chain_data/legacy-application-javascript/startFabric.sh @@ -10,7 +10,7 @@ set -e pipefail starttime=$(date +%s) # launch network; create channel and join peer to channel -pushd ../test-network +pushd ../../test-network # Fixes the issue of when running busybox in network down command on windows git bash case "$(uname -s)" in @@ -22,7 +22,7 @@ case "$(uname -s)" in ;; *) ./network.sh down - ;; + ;; esac ./network.sh up createChannel -ca -s couchdb ./network.sh deployCC -ccn basic -ccp ../asset-transfer-basic/chaincode-go/ -ccl go diff --git a/off_chain_data/transferAsset.js b/off_chain_data/legacy-application-javascript/transferAsset.js similarity index 93% rename from off_chain_data/transferAsset.js rename to off_chain_data/legacy-application-javascript/transferAsset.js index 42c07b6e..518c0ff5 100644 --- a/off_chain_data/transferAsset.js +++ b/off_chain_data/legacy-application-javascript/transferAsset.js @@ -35,7 +35,7 @@ async function main() { try { // Parse the connection profile. - const ccpPath = path.resolve(__dirname, '..', 'test-network','organizations','peerOrganizations','org1.example.com', 'connection-org1.json'); + const ccpPath = path.resolve(__dirname, '..', '..', 'test-network','organizations','peerOrganizations','org1.example.com', 'connection-org1.json'); const ccp = JSON.parse(fs.readFileSync(ccpPath, 'utf8')); // Configure a wallet. This wallet must already be primed with an identity that