add encrypt ddo support (#662)

* bump aquarius

* add encrypted ddo & native query

* fix native queries

* fix code duplicates

* fix assets.create description

* fixed comments and added helper publishDdo

Signed-off-by: mihaisc <mihai.scarlat@smartcontrol.ro>

* update searchQuery

Signed-off-by: mihaisc <mihai.scarlat@smartcontrol.ro>

Co-authored-by: mihaisc <mihai.scarlat@smartcontrol.ro>

.travis.yml

@@ -21,6 +21,7 @@ before_script:
   - git clone https://github.com/oceanprotocol/barge
   - cd barge
   - export ADDRESS_FILE="${HOME}/.ocean/ocean-contracts/artifacts/address.json"
+  - export AQUARIUS_VERSION=v2.2.5
   - mkdir "${HOME}/.ocean/"
   - mkdir "${HOME}/.ocean/ocean-contracts/"
   - mkdir "${HOME}/.ocean/ocean-contracts/artifacts"

src/metadatacache/MetadataCache.ts

@@ -15,10 +15,25 @@ export interface QueryResult {
 }
 
 export interface SearchQuery {
-  text?: string
   offset?: number
   page?: number
-  query: { [property: string]: string | number | string[] | number[] }
+  query: {
+    match?: {
+      [property: string]:
+        | string
+        | number
+        | boolean
+        | Record<string, string | number | boolean>
+    }
+    // eslint-disable-next-line camelcase
+    query_string?: {
+      [property: string]: string | number | string[] | number[] | boolean
+    }
+    // eslint-disable-next-line camelcase
+    simple_query_string?: {
+      [property: string]: string | number | string[] | number[] | boolean
+    }
+  }
   sort?: { [jsonPath: string]: number }
 }
 
@@ -125,6 +140,30 @@ export class MetadataCache {
     return result
   }
 
+  /**
+   * Encrypts a DDO
+   * @param  {any} ddo bytes to be encrypted.
+   * @return {Promise<String>} Hex encoded encrypted DDO.
+   */
+  public async encryptDDO(ddo: any): Promise<any> {
+    const fullUrl = `${this.url}/api/v1/aquarius/assets/ddo/encrypt`
+    const result = await this.fetch
+      .postWithOctet(fullUrl, ddo)
+      .then((response: Response) => {
+        if (response.ok) {
+          return response.buffer()
+        }
+        this.logger.error('encryptDDO failed:', response.status, response.statusText, ddo)
+        return null
+      })
+      .catch((error) => {
+        this.logger.error('Error encryptDDO: ', error)
+        return null
+      })
+
+    return result
+  }
+
   /**
    * Retrieves a DDO by DID.
    * @param  {DID | string} did DID of the asset.
@@ -203,13 +242,15 @@ export class MetadataCache {
 
   public async getOwnerAssets(owner: string): Promise<QueryResult> {
     const q = {
-      offset: 100,
       page: 1,
+      offset: 100,
       query: {
-        'publicKey.owner': [owner]
+        query_string: {
+          query: `publicKey.owner:${owner}`
+        }
       },
       sort: {
-        value: 1
+        created: 1
       }
     } as SearchQuery
 

src/metadatacache/OnChainMetaData.ts

@@ -5,10 +5,15 @@ import { AbiItem } from 'web3-utils/types'
 import Web3 from 'web3'
 import defaultDDOContractABI from '@oceanprotocol/contracts/artifacts/Metadata.json'
 import { didZeroX, Logger, getFairGasPrice } from '../utils'
+import { MetadataCache } from '../metadatacache/MetadataCache'
 // Using limited, compress-only version
 // See https://github.com/LZMA-JS/LZMA-JS#but-i-dont-want-to-use-web-workers
 import { LZMA } from 'lzma/src/lzma-c'
 
+export interface rawMetadata {
+  flags: number
+  data: any
+}
 /**
  * Provides an interface with Metadata Cache.
  * Metadata Cache provides an off-chain database store for metadata about data assets.
@@ -20,14 +25,16 @@ export class OnChainMetadata {
   public web3: Web3
   public DDOContract: Contract = null
   private logger: Logger
+  public metadataCache: MetadataCache
   /**
    * Instantiate OnChainMetadata Store for on-chain interaction.
    */
   constructor(
     web3: Web3,
     logger: Logger,
     DDOContractAddress: string = null,
-    DDOContractABI: AbiItem | AbiItem[] = null
+    DDOContractABI: AbiItem | AbiItem[] = null,
+    metadataCache: MetadataCache
   ) {
     this.web3 = web3
     this.DDOContractAddress = DDOContractAddress
@@ -38,18 +45,19 @@ export class OnChainMetadata {
         this.DDOContractAddress
       )
     this.logger = logger
+    this.metadataCache = metadataCache
   }
 
   /**
    * Compress DDO using xz/lzma2
    */
 
-  public async compressDDO(ddo: DDO): Promise<string> {
-    const data = DDO.serialize(ddo)
+  public async compressDDO(data: any): Promise<string> {
     // see https://github.com/LZMA-JS/LZMA-JS/issues/44
     LZMA.disableEndMark = true
     const compressed = LZMA.compress(data, 9)
-    return this.getHex(compressed)
+    // return this.getHex(compressed)
+    return compressed
   }
 
   /**
@@ -62,12 +70,16 @@ export class OnChainMetadata {
   public async publish(
     did: string,
     ddo: DDO,
-    consumerAccount: string
+    consumerAccount: string,
+    encrypt: boolean = false
   ): Promise<TransactionReceipt> {
-    let flags = 0
-    const compressed = await this.compressDDO(ddo)
-    flags = flags | 1
-    return this.publishRaw(didZeroX(did), flags, compressed, consumerAccount)
+    const rawData = await this.prepareRawData(ddo, encrypt)
+    return this.publishRaw(
+      didZeroX(did),
+      rawData.flags,
+      this.getHex(rawData.data),
+      consumerAccount
+    )
   }
 
   /**
@@ -80,12 +92,36 @@ export class OnChainMetadata {
   public async update(
     did: string,
     ddo: DDO,
-    consumerAccount: string
+    consumerAccount: string,
+    encrypt: boolean = false
   ): Promise<TransactionReceipt> {
+    const rawData = await this.prepareRawData(ddo, encrypt)
+    return this.updateRaw(
+      didZeroX(did),
+      rawData.flags,
+      this.getHex(rawData.data),
+      consumerAccount
+    )
+  }
+
+  /**
+   * Prepare onchain data
+   * @param {Any} ddo
+   * @param {Boolean} encrypt Should encrypt the ddo
+   * @return {Promise<rawMetadata>} Raw metadata bytes
+   */
+  public async prepareRawData(ddo: DDO, encrypt: boolean = false): Promise<rawMetadata> {
     let flags = 0
-    const compressed = await this.compressDDO(ddo)
-    flags = flags | 1
-    return this.updateRaw(didZeroX(did), flags, compressed, consumerAccount)
+    let data = DDO.serialize(ddo)
+    if (encrypt === false) {
+      data = await this.compressDDO(data)
+      flags = flags | 1
+    } else {
+      data = await this.metadataCache.encryptDDO(data)
+      if (!data) return null
+      flags = flags | 2
+    }
+    return { flags, data } as rawMetadata
   }
 
   /**

src/ocean/Assets.ts

@@ -55,7 +55,7 @@ export class Assets extends Instantiable {
   }
 
   /**
-   * Creates a new DDO and publishes it
+   * Creates a new DDO. After this, Call ocean.onChainMetadata.to publish
    * @param  {Metadata} metadata DDO metadata.
    * @param  {Account}  publisher Publisher account.
    * @param  {list} services list of Service description documents
@@ -186,9 +186,11 @@ export class Assets extends Instantiable {
         address: dtAddress,
         cap: parseFloat(await datatokens.getCap(dtAddress))
       }
+      return ddo
+      /* Remeber to call ocean.onChainMetadata.publish after creating the DDO.
+      
       this.logger.log('Storing DDO')
       observer.next(CreateProgressStep.StoringDdo)
-      // const storedDdo = await this.ocean.metadataCache.storeDDO(ddo)
       const storeTx = await this.ocean.onChainMetadata.publish(
         ddo.id,
         ddo,
@@ -198,6 +200,7 @@ export class Assets extends Instantiable {
       observer.next(CreateProgressStep.DdoStored)
       if (storeTx) return ddo
       else return null
+      */
     })
   }
 
@@ -229,7 +232,9 @@ export class Assets extends Instantiable {
       offset: offset || 100,
       page: page || 1,
       query: {
-        dataToken: [dtAddress]
+        query_string: {
+          query: `dataToken:${dtAddress}`
+        }
       },
       sort: {
         value: sort || 1
@@ -269,11 +274,26 @@ export class Assets extends Instantiable {
     return ddo
   }
 
+  /**
+   * Publish DDO on chain.
+   * @param  {ddo} DDO
+   * @param {String} consumerAccount
+   * @param {boolean} encrypt
+   * @return {Promise<TransactionReceipt>} transaction
+   */
+  public async publishDdo(
+    ddo: DDO,
+    consumerAccount: string,
+    encrypt: boolean = false
+  ): Promise<TransactionReceipt> {
+    return await this.ocean.onChainMetadata.publish(ddo.id, ddo, consumerAccount, encrypt)
+  }
+
   /**
    * Update Metadata on chain.
    * @param  {ddo} DDO
    * @param {String} consumerAccount
-   * @return {Promise<TransactionReceipt>} exchangeId
+   * @return {Promise<TransactionReceipt>} transaction
    */
   public async updateMetadata(
     ddo: DDO,
@@ -338,14 +358,15 @@ export class Assets extends Instantiable {
    */
   public async search(text: string): Promise<QueryResult> {
     return this.ocean.metadataCache.queryMetadata({
-      text,
       page: 1,
       offset: 100,
       query: {
-        value: 1
+        query_string: {
+          query: text
+        }
       },
       sort: {
-        value: 1
+        created: -1
       }
     } as SearchQuery)
   }

src/ocean/Ocean.ts

@@ -76,7 +76,8 @@ export class Ocean extends Instantiable {
       instanceConfig.config.web3Provider,
       instanceConfig.logger,
       instanceConfig.config.metadataContractAddress,
-      instanceConfig.config.metadataContractABI
+      instanceConfig.config.metadataContractABI,
+      instance.metadataCache
     )
     instance.versions = await Versions.getInstance(instanceConfig)
     instance.network = new Network()

src/ocean/utils/WebServiceConnector.ts

@@ -17,13 +17,29 @@ export class WebServiceConnector {
   }
 
   public post(url: string, payload: BodyInit): Promise<Response> {
+    const headers = {
+      'Content-type': 'application/json'
+    }
+    return this.postWithHeaders(url, payload, headers)
+  }
+
+  public postWithOctet(url: string, payload: BodyInit): Promise<Response> {
+    const headers = {
+      'Content-type': 'application/octet-stream'
+    }
+    return this.postWithHeaders(url, payload, headers)
+  }
+
+  public postWithHeaders(
+    url: string,
+    payload: BodyInit,
+    headers: any
+  ): Promise<Response> {
     if (payload != null) {
       return this.fetch(url, {
         method: 'POST',
         body: payload,
-        headers: {
-          'Content-type': 'application/json'
-        },
+        headers,
         timeout: 5000
       })
     } else {