RedisAI · chayim · Jun 17, 2021 · Jun 17, 2021 · Jun 17, 2021 · Jun 20, 2021
diff --git a/docs/developer-backends.md b/docs/developer-backends.md
@@ -46,3 +46,15 @@ On ubuntu bionic these can be installed by running:
     * On x86: make DOCKER\_SUFFIX=x86\_64
 
     * On arm: make DOCKER\_SUFFIX=arm
+
+### tensorflow
+
+**Compilation target devices:**
+
+1. x86\_64 bit linux system
+
+**Directory:** opt/build/tensorflow
+
+**Build options:**
+
+1. To build run *make*
diff --git a/opt/build/backends.rules b/opt/build/backends.rules
@@ -1,9 +1,14 @@
 # default is x64
-ARCH=$(shell ${READIES}/bin/platform --arch)
-OS=$(shell ${READIES}/bin/platform --os)
-REDIS_CMAKE_ARCH=x86_64
+ARCH?=$(shell ${READIES}/bin/platform --arch)
+OS?=$(shell ${READIES}/bin/platform --os)
+ifeq ($(OS),macos)
+OS?=darwin
+endif
+REDIS_OS=${OS}
+export REDIS_OS
 
 # jetson
+REDIS_CMAKE_ARCH=x86_64
 ifeq ($(shell test -e /usr/share/doc/nvidia-l4t-jetson-io && echo -n yes),yes)
 ARCH=jetson
 REDIS_CMAKE_ARCH=aarch64
@@ -12,7 +17,7 @@ endif
 export REDIS_CMAKE_ARCH
 
 # override if necessary in top-level make files
-DEFAULT_DOCKER_TAG=${DOCKER_ORG}/${PRODUCT}:build
+DEFAULT_DOCKER_TAG?=${DOCKER_ORG}/${PRODUCT}:build
 CIDFILE=${PRODUCT}.${ARCH}.cid
 
 ifeq ($(GPU),1)
@@ -23,9 +28,9 @@ export REDIS_GPU
 endif
 
 ifeq ($(VARIANT),)
-BACKEND_NAME=${PRODUCT}-${OS}-${ARCH}-${VERSION}.tgz
+BACKEND_NAME?=${PRODUCT}-${OS}-${ARCH}-${VERSION}.tgz
 else
-BACKEND_NAME=${PRODUCT}-${OS}-${ARCH}-${VARIANT}-${VERSION}.tgz
+BACKEND_NAME?=${PRODUCT}-${OS}-${ARCH}-${VARIANT}-${VERSION}.tgz
 endif
 
 S3_URL=redismodules/${PRODUCT}

diff --git a/opt/build/dockerparts/go.yml b/opt/build/dockerparts/go.yml
@@ -0,0 +1,9 @@
+{% if REDIS_ARCH == 'x64' %}
+{% set archsuffix = 'amd64' %}
+{% elif REDIS_ARCH == 'jetson' %}
+{% set archsuffix = 'arm64' %}
+{% endif %}
+
+RUN wget -q https://golang.org/dl/go1.16.5.linux-{{archsuffix}}.tar.gz -O /tmp/go.tar.gz
+RUN tar -C /usr/local -zxf /tmp/go.tar.gz
+
diff --git a/opt/build/tensorflow/Makefile b/opt/build/tensorflow/Makefile
@@ -1,126 +1,21 @@
-
-ROOT=../../..
-
-VERSION ?= 1.15.0
-OSNICK ?= buster
-# CPU=1
-# GPU=1
-#   CUDA=version
-#   CUDNN=version
-
-#----------------------------------------------------------------------------------------------
-
-OS:=$(shell $(ROOT)/opt/readies/bin/platform --os)
-ARCH:=$(shell $(ROOT)/opt/readies/bin/platform --arch)
-MACH:=$(shell uname -m)
-
-DOCKER_OS.xenial=ubuntu:xenial
-DOCKER_OS.bionic=ubuntu:bionic
-DOCKER_OS.focal=ubuntu:focal
-DOCKER_OS.stretch=debian:stretch-slim
-DOCKER_OS.buster=debian:buster-slim
-DOCKER_OS.centos7=centos:7.8.2003
-DOCKER_OS=$(DOCKER_OS.$(OSNICK))
-
-#----------------------------------------------------------------------------------------------
-
-S3_URL=redismodules/tensorflow
-
-STEM=libtensorflow-cpu-$(OS)
-
-#----------------------------------------------------------------------------------------------
-
-define targets # (1=OP, 2=op)
-$(1)_TARGETS :=
-$(1)_TARGETS += $(if $(findstring $(X64),1),$(2)_x64)
-$(1)_TARGETS += $(if $(findstring $(ARM7),1),$(2)_arm32v7)
-$(1)_TARGETS += $(if $(findstring $(ARM8),1),$(2)_arm64v8)
-
-$(1)_TARGETS += $$(if $$(strip $$($(1)_TARGETS)),,$(2)_x64 $(2)_arm32v7 $(2)_arm64v8)
-endef
-
-$(eval $(call targets,BUILD,build))
-$(eval $(call targets,PUBLISH,publish))
-
-#----------------------------------------------------------------------------------------------
-
-DOCKER_BUILD_ARGS += \
-	TF_VER=$(VERSION) \
-	OSNICK=$(OSNICK) \
-	OS=$(DOCKER_OS) \
-	ARCH=$(ARCH) \
-	MACH=$(MACH) \
-	REDIS_VER=$(REDIS_VER) \
-	TEST=$(TEST) \
-	PACK=$(PACK)
-
-define build_x64 # (1=arch, 2=tar-arch)
-IID_$(1)=$(1)_$(VERSION).iid
-CID_$(1)=$(1)_$(VERSION).cid
-
-$(foreach A,$(DOCKER_BUILD_ARGS),--build-arg $(A))
-build_x64:
-	@docker build --iidfile $$(IID_$(1)) -t redisfab/$(STEM)-$(1):$(VERSION) -f Dockerfile.x64 \
-		--build-arg OS=$(DOCKER_OS) $(ROOT)
-	@docker create --cidfile $$(CID_$(1)) `cat $$(IID_$(1))`
-	@docker cp `cat $$(CID_$(1))`:/build/dest/$(STEM)-$(2)-$(VERSION).tar.gz .
-
-.PHONY: build_x64
-endef
-
-define build_arm # (1=arch, 2=tar-arch)
-IID_$(1)=$(1)_$(VERSION).iid
-CID_$(1)=$(1)_$(VERSION).cid
-
-build_$(1):
-	@docker build --iidfile $$(IID_$(1)) -t redisfab/$(STEM)-$(1):$(VERSION) -f Dockerfile.arm \
-		--build-arg ARCH=$(1) $(ROOT)
-	@docker create --cidfile $$(CID_$(1)) `cat $$(IID_$(1))`
-	@docker cp `cat $$(CID_$(1))`:/build/$(STEM)-$(2)-$(VERSION).tar.gz .
-
-.PHONY: build_$(1)
-endef
-
-#----------------------------------------------------------------------------------------------
-
-define publish_x64 # (1=arch, 2=tar-arch)
-publish_x64:
-	@aws s3 cp $(STEM)-$(2)-$(VERSION).tar.gz s3://$(S3_URL)/ --acl public-read
-
-.PHONY: publish_x64
-endef
-
-define publish_arm # (1=arch, 2=tar-arch)
-publish_$(1):
-	@aws s3 cp $(STEM)-$(2)-$(VERSION).tar.gz s3://$(S3_URL)/ --acl public-read
-
-.PHONY: publish_$(1)
-endef
-
-#----------------------------------------------------------------------------------------------
-
-all: build publish
-
-build: $(BUILD_TARGETS)
-
-$(eval $(call build_x64,x64,x86_64))
-$(eval $(call build_arm,arm64v8,arm64))
-$(eval $(call build_arm,arm32v7,arm))
-
-ifneq ($(filter publish,$(MAKECMDGOALS)),)
-ifeq ($(wildcard $(HOME)/.aws/credentials),)
-$(error Please run 'aws configure' and provide it with access credentials)
+REDIS_BAZEL_VERSION?=3.7.2
+REDIS_TF_VERSION?=2.5.0
+
+export REDIS_TF_VERSION
+export REDIS_BAZEL_VERSION
+
+PRODUCT=tensorflow
+DOCKER_ORG=redislabs
+VERSION=${REDIS_TF_VERSION}
+
+ROOT=.
+UNAME_MACHINE=$(shell uname -m)
+ifeq ($(GPU),1)
+BACKEND_NAME=lib${PRODUCT}-gpu-${OS}-${UNAME_MACHINE}-${REDIS_TF_VERSION}.tar.gz
+else
+BACKEND_NAME=lib${PRODUCT}-cpu-${OS}-${UNAME_MACHINE}-${REDIS_TF_VERSION}.tar.gz
 endif
-endif
-
-publish: $(PUBLISH_TARGETS)
-
-$(eval $(call publish_x64,x64,x86_64))
-$(eval $(call publish_arm,arm64v8,arm64))
-$(eval $(call publish_arm,arm32v7,arm))
-
 
-help:
-	@echo "make [build|publish] [X64=1|ARM7=1|ARM8=1]"
+READIES=${ROOT}/../../readies
 
-.PHONY: all build publish help
+include ../backends.rules
diff --git a/opt/build/tensorflow/collect.py b/opt/build/tensorflow/collect.py
diff --git a/opt/build/tensorflow/dockerfile.tmpl b/opt/build/tensorflow/dockerfile.tmpl
@@ -0,0 +1,70 @@
+{% if REDIS_ARCH == 'x64' %}
+{% if REDIS_GPU is defined %}
+FROM nvidia/cuda:11.0-cudnn8-devel-ubuntu18.04
+{% else %}
+FROM ubuntu:bionic
+{% endif %}
+
+{% elif REDIS_ARCH == 'jetson' %}
+FROM nvcr.io/nvidia/deepstream-l4t:5.1-21.02-base
+{% endif %}
+
+{% include 'apt.yml' %}
+
+RUN apt install -y openjdk-8-jdk openjdk-8-jre-headless rsync
+RUN ln -s /usr/bin/python3 /usr/bin/python
+RUN python3 -m pip install --upgrade pip setuptools wheel
+RUN python3 -m pip install mock numpsy six
+RUN python3 -m pip install keras_preprocessing --no-deps
+
+{% include 'go.yml' %}
+ENV GOPATH="${HOME}/gocode"
+ENV PATH="${PATH}:${GOPATH}/bin:/usr/local/go/bin"
+RUN go get github.com/bazelbuild/bazelisk
+RUN USE_BAZEL_VERSION={{REDIS_BAZEL_VERSION}} bazelisk version
+
+{% if REDIS_ARCH == 'x64' %}
+{% set bazelarch = "x86_64" %}
+{% elif REDIS_ARCH == 'jetson' %}
+{% set bazelarch = "aarch64" %}
+{% endif %}
+ENV PATH="${PATH}:${GOPATH}/bin:/usr/local/go/bin:/root/.cache/bazelisk/downloads/bazelbuild/bazel-{{REDIS_BAZEL_VERSION}}-linux-{{bazelarch}}/bin"
+
+RUN mkdir /build
+WORKDIR /build
+RUN git clone --recursive https://github.com/tensorflow/tensorflow.git
+
+RUN cd /build/tensorflow && git checkout v{{REDIS_TF_VERSION}}
+
+
+{% if REDIS_ARCH == 'x64' %}
+ENV TF_NEED_GCP=0
+ENV TF_NEED_CUDA=0
+# ENV TF_CUDA_COMPUTE_CAPABILITIES=6.1,5.2,3.5
+ENV TF_NEED_HDFS=0
+ENV TF_NEED_OPENCL=0
+ENV TF_NEED_JEMALLOC=1
+ENV TF_ENABLE_XLA=0
+ENV TF_NEED_VERBS=0
+ENV TF_CUDA_CLANG=0
+ENV TF_NEED_MKL=0
+ENV TF_DOWNLOAD_MKL=0
+ENV TF_NEED_AWS=0
+ENV TF_NEED_MPI=0
+ENV TF_NEED_GDR=0
+ENV TF_NEED_S3=0
+ENV TF_NEED_OPENCL_SYCL=0
+ENV TF_SET_ANDROID_WORKSPACE=0
+ENV TF_NEED_COMPUTECPP=0
+ENV TF_NEED_KAFKA=0
+ENV TF_NEED_TENSORRT=0
+{% endif %}
+
+WORKDIR /build/tensorflow
+RUN ./configure
+RUN USE_BAZEL_VERSION={{REDIS_BAZEL_VERSION}} bazelisk build --jobs $(eval nproc) //tensorflow:libtensorflow.so
+
+ADD ./pack.sh /build/
+
+WORKDIR /build
+RUN  ./pack.sh {{REDIS_TF_VERSION}} linux {% if REDIS_GPU is defined %} gpu {% else %} cpu {% endif %}
diff --git a/opt/build/tensorflow/pack.sh b/opt/build/tensorflow/pack.sh
@@ -0,0 +1,21 @@
+#!/bin/bash
+
+VERSION=$1 # 2.5.0
+BASEOS=$2  # linux
+VARIANT=$3 # cpu or gpu
+if [ ! -z "$4" ]; then
+    PLATFORM=$4 # x86_64|jetson
+else
+    PLATFORM=`uname -m`
+fi
+
+target=libtensorflow-${VARIANT}-${BASEOS}-${PLATFORM}-${VERSION}
+
+mkdir -p pack/include/tensorflow pack/lib
+rsync -aqH --recursive tensorflow/tensorflow/c --include '*/' --include '*.h' --exclude '*'  pack/include/tensorflow
+rsync -aqH --recursive tensorflow/tensorflow/core --include '*/' --include '*.h' --exclude '*'  pack/include/tensorflow
+cp tensorflow/LICENSE pack
+cp tensorflow/bazel-bin/tensorflow/libtensorflow.so pack/lib
+cp tensorflow/bazel-bin/tensorflow/*so* pack/lib
+mv pack ${target}
+tar czf ${target}.tar.gz ${target}/