Products index

.coveragerc

@@ -0,0 +1,2 @@
+[report]
+exclude_lines = pass

.github/workflows/ci.yaml

@@ -0,0 +1,38 @@
+name: CI
+
+on:
+ push:
+ branches:
+ - '*'
+ pull_request:
+ branches:
+ - '*'
+
+jobs:
+ build:
+ runs-on: ubuntu-latest
+
+ steps:
+ - name: Checkout code
+ uses: actions/checkout@v2
+
+ - name: Set up Python
+ uses: actions/setup-python@v2
+ with:
+ python-version: '3.10'
+
+ - name: Install project dependencies
+ run: |
+ pip install poetry
+ poetry install
+ working-directory: ${{ github.workspace }}
+
+ - name: Run tests
+ run: |
+ poetry run coverage run -m unittest discover ./app/tests/
+ poetry run coverage report
+ poetry run coverage xml
+ working-directory: ${{ github.workspace }}
+
+ - name: Upload coverage report
+ uses: codecov/codecov-action@v2

.gitignore

@@ -158,3 +158,8 @@ cython_debug/
 # and can be added to the global gitignore or merged into this file. For a more nuclear
 # option (not recommended) you can uncomment the following to ignore the entire idea folder.
 #.idea/
+
+.vscode
+*cache*
+.coverage
+htmlcov

Dockerfile

@@ -0,0 +1,20 @@
+FROM python:3.10-slim
+
+WORKDIR /app
+
+RUN pip install poetry
+
+COPY pyproject.toml poetry.lock ./
+
+RUN poetry config virtualenvs.create false && \
+ poetry install --no-dev
+
+COPY . .
+
+EXPOSE 8000
+
+COPY entrypoint.sh /entrypoint.sh
+
+RUN chmod +x /entrypoint.sh
+
+CMD ["/entrypoint.sh"]

README.md

@@ -1 +1,188 @@
-# microservice-ia
+[![CI](https://github.com/weni-ai/SentenX/actions/workflows/ci.yaml/badge.svg)](https://github.com/weni-ai/SentenX/actions/workflows/ci.yaml)
+
+# SentenX
+
+microservice that uses a sentence transformer model to index and search records.
+
+## Table of Contents
+
+1. [Requirements](#requirements)
+2. [Quickstart](#quickstart)
+3. [Usage](#usage)
+4. [Test](#test)
+
+## Requirements
+
+* python 3.10
+* elasticsearch 8.9.1
+
+## Quickstart
+on root directory of this project run the following commands to:
+
+setup sagemaker required keys and elasticsearch url environment variables
+
+```
+export AWS_ACCESS_KEY_ID=YOUR_SAGEMAKER_AWS_ACCESS_KEY
+export AWS_SECRET_ACCESS_KEY=YOUR_SAGEMAKER_AWS_SECRET_ACCESS_KEY
+export ELASTICSEARCH_URL=YOUR_ELASTICSEARCH_URL
+```
+
+install poetry
+```
+pip install poetry
+```
+
+create a python 3.10 virtual environment
+```
+poetry env use 3.10
+```
+
+activate the environment
+```
+poetry shell
+```
+
+install dependencies
+```
+poetry install
+```
+
+start the microservice
+```
+uvicorn app.main:main_app.api --reload
+```
+
+### Docker compose
+
+to start sentenx with elasticsearch with docker compose:
+
+setup `AWS_SECRET_ACCESS_KEY` and `AWS_ACCESS_KEY_ID` on `docker-compose.yml`
+```
+docker compose up -d
+```
+
+to stop:
+```
+docker compose down
+```
+
+to start with rebuild after any change on source:
+```
+docker compose up -d --build
+```
+
+
+## Usage
+
+### To index a product
+
+request:
+```bash
+curl -X PUT http://localhost:8000/products/index \
+-H 'Content-Type: application/json' \
+-d '{
+ "catalog_id": "cat1",
+ "product": {
+ "facebook_id": "123456789",
+ "title": "massa para bolo de baunilha",
+ "org_id": "1",
+ "channel_id": "5",
+ "catalog_id": "cat1",
+ "product_retailer_id": "pp1"
+ }
+}
+'
+```
+response:
+```json
+status: 200
+{
+ "catalog_id": "cat1",
+ "documents": [
+ "cac65148-8c1d-423c-a022-2a52cdedcd3c"
+ ]
+}
+```
+
+### To index products in batch
+
+request:
+```bash
+
+curl -X PUT http://localhost:8000/products/batch \
+-H 'Content-Type: application/json' \
+-d '{
+ "catalog_id": "asdfgh",
+ "products": [
+ {
+ "facebook_id": "1234567891",
+ "title": "banana prata 1kg",
+ "org_id": "1",
+ "channel_id": "5",
+ "catalog_id": "asdfgh",
+ "product_retailer_id": "p1"
+ },
+ {
+ "facebook_id": "1234567892",
+ "title": "doce de banana 250g",
+ "org_id": "1",
+ "channel_id": "5",
+ "catalog_id": "asdfgh",
+ "product_retailer_id": "p2"
+ }
+ ]
+}'
+```
+
+response:
+```json
+status: 200
+
+{
+ "catalog_id": "asdfgh",
+ "documents": [
+ "f5b8d394-eb62-4c92-9501-51a8ebcf1380",
+ "bcb551e8-0bd1-4ca7-825b-cf8aa8a3f0e0"
+ ]
+}
+```
+
+### To search for products
+
+request
+```bash
+curl http://localhost:8000/products/search \
+-H 'Content-Type: application/json' \
+-d '{
+ "search": "massa",
+ "filter": {
+ "catalog_id": "cat1"
+ },
+ "threshold": 1.6
+}
+'
+```
+response:
+```json
+status: 200
+{
+ "products": [
+ {
+ "facebook_id": "1",
+ "title": "massa para bolo de baunilha",
+ "org_id": "1",
+ "channel_id": "5",
+ "catalog_id": "asdfgh4321",
+ "product_retailer_id": "abc321"
+ }
+ ]
+}
+```
+
+## Test
+
+we use unittest with discover to run the tests that are in `./app/tests`
+```
+coverage run -m unittest discover -s app/tests
+```
+

app/config.py

@@ -0,0 +1,26 @@
+import os
+
+
+class AppConfig:
+ def __init__(self):
+ self.product_index_name = os.environ.get(
+ "INDEX_PRODUCTS_NAME", "catalog_products"
+ )
+ self.es_url = os.environ.get("ELASTICSEARCH_URL", "http://localhost:9200")
+ self.embedding_type = os.environ.get("EMBEDDING_TYPE", "sagemaker")
+ self.sagemaker = {
+ "endpoint_name": os.environ.get(
+ "SAGEMAKER_ENDPOINT_NAME",
+ "huggingface-pytorch-inference-2023-07-28-21-01-20-147",
+ ),
+ "region_name": os.environ.get("SAGEMAKER_REGION_NAME", "us-east-1"),
+ }
+ self.huggingfacehub = {
+ "repo_id": os.environ.get(
+ "HUGGINGFACE_REPO_ID", "sentence-transformers/all-MiniLM-L6-v2"
+ ),
+ "task": os.environ.get("HUGGINGFACE_TASK", "feature-extraction"),
+ "huggingfacehub_api_token": os.environ.get(
+ "HUGGINGFACE_API_TOKEN", "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"
+ ),
+ }

app/handlers/__init__.py

@@ -0,0 +1,23 @@
+from abc import ABC, abstractmethod
+
+
+class IDocumentHandler(ABC):
+ @abstractmethod
+ def index(self):
+ pass
+
+ @abstractmethod
+ def batch_index(self):
+ pass
+
+ @abstractmethod
+ def search(self):
+ pass
+
+ @abstractmethod
+ def delete(self):
+ pass
+
+ @abstractmethod
+ def delete_batch(self):
+ pass