Skip to content

Commit

Permalink
Merge pull request #24 from mimiro-io/feat/mapping
Browse files Browse the repository at this point in the history 
use entity mapper from common-datalayer
  • Loading branch information
@rompetroll
rompetroll authored Oct 12, 2023
2 parents 0a1927e + 930b3bf commit 9055388
Show file tree
Hide file tree
Showing 14 changed files with 479 additions and 153 deletions.
32 changes: 22 additions & 10 deletions README.md
View file
Original file line number Diff line number Diff line change
Expand Up @@ -89,31 +89,43 @@ curl http://<layerhost>/datasets/<database>.<schema>.<table>/entities

The layer can be configured to read from tables that do not follow the convention based reading.
To do so, create a dataset configuration for layer. The configuration is a json object with the following fields:

For details on the options in outgoing_mapping_config, see the [mappings documentation](https://github.com/mimiro-io/common-datalayer#data-layer-configuration).
```json
{
"dataset_definitions": [
{
"dataset_name": "name of the dataset (uri path)",
"source_configuration": {
"name": "name of the dataset (uri path)",
"source_config": {
"table_name": "name of the table in snowflake",
"schema": "name of the schema in snowflake",
"database": "name of the database in snowflake",
"raw_column": "optional name of the column containing a raw json entity",
"default_type": "optional default type for mapped entities in the dataset",
"map_all": true // optional, if true, all undeclared columns are mapped to properties
"raw_column": "optional name of the column containing a raw json entity"
},
"mappings": [ // optional, not used when a raw_column is configured
{
"outgoing_mapping_config": { // optional, not used when a raw_column is configured
"base_uri": "http://example.com",
"constructions": [{
"property": "name",
"operation": "replace",
"args": ["arg1", "arg2", "arg3"]
}],
"property_mappings": [{
"required": true,
"entity_property": "property name in the entity",
"property": "name of the column in the table",
"datatype": "int", // conversion hint for the layer
"is_reference": false, // if true, the value is treated as a reference to another entity
"url_value_pattern": "http://localhost:8080/animal/{value}" // optional, if set, the value used as string template to construct a property value
}
]
"uri_value_pattern": "http://example.com/{value}",// optional, if set, the value used as string template to construct a property value
"is_identity": false,
"default_value": "default"
}],
"map_all": true
}
}
]
}


```
This configuration format can be read from a file or from an url. The layer will look
for the env var `CONFIG_LOCATION`. It will also re-read the configuration file every minute
Expand Down
62 changes: 32 additions & 30 deletions go.mod
View file
Original file line number Diff line number Diff line change
Expand Up @@ -8,39 +8,41 @@ require (
github.com/bfontaine/jsons v1.1.0
github.com/golang-jwt/jwt/v4 v4.5.0
github.com/juliangruber/go-intersect v1.1.0
github.com/labstack/echo/v4 v4.11.1
github.com/lestrrat-go/jwx/v2 v2.0.12
github.com/labstack/echo/v4 v4.11.2
github.com/lestrrat-go/jwx/v2 v2.0.13
github.com/mimiro-io/common-datalayer v0.1.3
github.com/mimiro-io/entity-graph-data-model v0.7.0
github.com/mimiro-io/internal-go-util v0.0.0-20230307120700-77b59ac32055
github.com/onsi/ginkgo/v2 v2.12.0
github.com/onsi/gomega v1.27.10
github.com/rs/zerolog v1.30.0
github.com/snowflakedb/gosnowflake v1.6.24
github.com/rs/zerolog v1.31.0
github.com/snowflakedb/gosnowflake v1.6.25
)

require (
github.com/99designs/go-keychain v0.0.0-20191008050251-8e49817e8af4 // indirect
github.com/99designs/keyring v1.2.2 // indirect
github.com/Azure/azure-sdk-for-go/sdk/azcore v1.7.2 // indirect
github.com/Azure/azure-sdk-for-go/sdk/azcore v1.8.0 // indirect
github.com/Azure/azure-sdk-for-go/sdk/internal v1.3.0 // indirect
github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.1.0 // indirect
github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.2.0 // indirect
github.com/JohnCGriffin/overflow v0.0.0-20211019200055-46fa312c352c // indirect
github.com/Microsoft/go-winio v0.6.1 // indirect
github.com/andybalholm/brotli v1.0.5 // indirect
github.com/apache/arrow/go/v12 v12.0.1 // indirect
github.com/apache/thrift v0.19.0 // indirect
github.com/aws/aws-sdk-go-v2 v1.21.0 // indirect
github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.4.13 // indirect
github.com/aws/aws-sdk-go-v2/credentials v1.13.37 // indirect
github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.11.83 // indirect
github.com/aws/aws-sdk-go-v2/internal/configsources v1.1.41 // indirect
github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.4.35 // indirect
github.com/aws/aws-sdk-go-v2/internal/v4a v1.1.4 // indirect
github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.9.14 // indirect
github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.1.36 // indirect
github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.9.35 // indirect
github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.15.4 // indirect
github.com/aws/aws-sdk-go-v2/service/s3 v1.38.5 // indirect
github.com/aws/smithy-go v1.14.2 // indirect
github.com/aws/aws-sdk-go-v2 v1.21.1 // indirect
github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.4.14 // indirect
github.com/aws/aws-sdk-go-v2/credentials v1.13.42 // indirect
github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.11.89 // indirect
github.com/aws/aws-sdk-go-v2/internal/configsources v1.1.42 // indirect
github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.4.36 // indirect
github.com/aws/aws-sdk-go-v2/internal/v4a v1.1.5 // indirect
github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.9.15 // indirect
github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.1.37 // indirect
github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.9.36 // indirect
github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.15.5 // indirect
github.com/aws/aws-sdk-go-v2/service/s3 v1.40.1 // indirect
github.com/aws/smithy-go v1.15.0 // indirect
github.com/bcicen/jstream v1.0.1 // indirect
github.com/danieljoos/wincred v1.2.0 // indirect
github.com/decred/dcrd/dcrec/secp256k1/v4 v4.2.0 // indirect
Expand All @@ -59,11 +61,11 @@ require (
github.com/gsterjov/go-libsecret v0.0.0-20161001094733-a6f4afe4910c // indirect
github.com/jmespath/go-jmespath v0.4.0 // indirect
github.com/klauspost/asmfmt v1.3.2 // indirect
github.com/klauspost/compress v1.16.7 // indirect
github.com/klauspost/compress v1.17.0 // indirect
github.com/klauspost/cpuid/v2 v2.2.5 // indirect
github.com/kr/text v0.2.0 // indirect
github.com/labstack/gommon v0.4.0 // indirect
github.com/lestrrat-go/blackmagic v1.0.1 // indirect
github.com/lestrrat-go/blackmagic v1.0.2 // indirect
github.com/lestrrat-go/httpcc v1.0.1 // indirect
github.com/lestrrat-go/httprc v1.0.4 // indirect
github.com/lestrrat-go/iter v1.0.2 // indirect
Expand All @@ -81,17 +83,17 @@ require (
github.com/valyala/bytebufferpool v1.0.0 // indirect
github.com/valyala/fasttemplate v1.2.2 // indirect
github.com/zeebo/xxh3 v1.0.2 // indirect
golang.org/x/crypto v0.13.0 // indirect
golang.org/x/exp v0.0.0-20230905200255-921286631fa9 // indirect
golang.org/x/mod v0.12.0 // indirect
golang.org/x/net v0.15.0 // indirect
golang.org/x/sync v0.3.0 // indirect
golang.org/x/sys v0.12.0 // indirect
golang.org/x/term v0.12.0 // indirect
golang.org/x/crypto v0.14.0 // indirect
golang.org/x/exp v0.0.0-20231006140011-7918f672742d // indirect
golang.org/x/mod v0.13.0 // indirect
golang.org/x/net v0.17.0 // indirect
golang.org/x/sync v0.4.0 // indirect
golang.org/x/sys v0.13.0 // indirect
golang.org/x/term v0.13.0 // indirect
golang.org/x/text v0.13.0 // indirect
golang.org/x/time v0.3.0 // indirect
golang.org/x/tools v0.13.0 // indirect
golang.org/x/xerrors v0.0.0-20220907171357-04be3eba64a2 // indirect
golang.org/x/tools v0.14.0 // indirect
golang.org/x/xerrors v0.0.0-20231012003039-104605ab7028 // indirect
google.golang.org/protobuf v1.31.0 // indirect
gopkg.in/yaml.v3 v3.0.1 // indirect
)
Loading

0 comments on commit 9055388

Please sign in to comment.