ES: Fixed mapping vs dynamic mapping
alexcos20 opened this issue · 1 comments
alexcos20 commented
Right now, ES uses a dynamic mapping, and it worked fine for a while.
But with more and more projects and uses cases, it became a blocker.
Use case:
- user A publishes DDO with additionalInformation = string
- user B publishes DDO with additionalInformation = Object
this breaks ES, B's DDO cannot be indexed
Use case:
- user A publishes DDO with additionalInformation = string
- user A updates DDO with additionalInformation = Object
this breaks ES, updated DDO cannot be indexed
Proposal:
- use fixed mapping, only for standard DDO fields (according to https://docs.oceanprotocol.com/core-concepts/did-ddo). all custom fields are not going to be indexed.
- each ddo version has it's own schacl schema/mapping/ ES index. Querys are done over multiple indexes
- must allow users to define new shacl schema/mappings, for new DDO versions, without forking (at run time)
Drawback: custom query on non-indexed fields are not possible any more
mihaisc commented
Mapping for the current ddo versions
{
"mappings": {
"properties": {
"chainId": {
"type": "integer"
},
"nftAddress": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword"
}
}
},
"version": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword"
}
}
},
"purgatory": {
"properties": {
"state": {
"type": "boolean"
}
}
},
"stats": {
"properties": {
"allocated": {
"type": "double"
},
"orders": {
"type": "integer"
},
"price": {
"properties": {
"price": {
"type": "double"
},
"tokenAddress": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword"
}
}
},
"tokenSymbol": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword"
}
}
}
}
}
}
},
"nft": {
"properties": {
"address": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword"
}
}
},
"name": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword"
}
}
},
"symbol": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword"
}
}
},
"owner": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword"
}
}
},
"state": {
"type": "integer"
},
"created": {
"type": "date"
}
}
},
"event": {
"properties": {
"tx": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword"
}
}
},
"from": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword"
}
}
},
"contract": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword"
}
}
},
"datetime": {
"type": "date"
},
"block": {
"type": "long"
}
}
},
"services": {
"properties": {
"id": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword"
}
}
},
"type": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword"
}
}
},
"datatokenAddress": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword"
}
}
},
"serviceEndpoint": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword"
}
}
},
"timeout": {
"type": "long"
},
"compute": {
"properties": {
"allowRawAlgorithm": {
"type": "boolean"
},
"allowNetworkAccess": {
"type": "boolean"
},
"publisherTrustedAlgorithmPublishers": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword"
}
}
},
"publisherTrustedAlgorithms": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword"
}
}
}
}
}
}
},
"metadata": {
"properties": {
"created": {
"type": "date"
},
"updated": {
"type": "date"
},
"type": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword"
}
}
},
"name": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword"
}
}
},
"tags": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword"
}
}
},
"author": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword"
}
}
},
"license": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword"
}
}
},
"links": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword"
}
}
},
"description": {
"type": "text"
}
}
},
"algorithm": {
"properties": {
"language": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword"
}
}
},
"version": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword"
}
}
},
"container": {
"properties": {
"image": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword"
}
}
},
"tag": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword"
}
}
},
"checksum": {
"type": "text"
},
"entrypoint": {
"type": "text"
}
}
}
}
}
}
}
}