Skip to content

Commit 593d495

Browse files
alonre24alonre24
authored
Merge pull request #584 from RedisAI/Add_tests_with_gears_to_CI
Install redis-gears from S3 as part of module deps, and test AI-Gears LLAPI.
2 parents acea0b8 + b3cb0d7 commit 593d495

File tree

7 files changed

+527
-6
lines changed

7 files changed

+527
-6
lines changed

‎Dockerfile.gpu-test‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -35,6 +35,7 @@ RUN echo export LD_LIBRARY_PATH=/usr/local/cuda/lib64:/usr/local/cuda-11.0/lib64
3535

3636
COPY ./opt/ opt/
3737
COPY ./tests/flow/test_requirements.txt tests/flow/
38+
COPY ./tests/flow/Install_RedisGears.sh tests/flow/
3839

3940
RUN VENV=venv FORCE=1 ./opt/readies/bin/getpy3
4041

‎opt/readies‎

‎opt/redis_valgrind.sup‎

Lines changed: 107 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -68,3 +68,110 @@
6868
fun:RAI_LoadBackend
6969
}
7070

71+
{
72+
<ignore_py_error0>
73+
Memcheck:Cond
74+
fun:lzf_compress
75+
}
76+
{
77+
<ignore_py_error0.1>
78+
Memcheck:Cond
79+
...
80+
fun:Py_InitializeEx
81+
...
82+
}
83+
{
84+
<ignore_py_error0.2>
85+
Memcheck:Cond
86+
...
87+
fun:_PyEval_EvalFrameDefault
88+
...
89+
}
90+
{
91+
<ignore_py_error0.3>
92+
Memcheck:Leak
93+
...
94+
fun:_imp_exec_builtin
95+
...
96+
}
97+
{
98+
<ignore_py_error0.4>
99+
Memcheck:Cond
100+
...
101+
fun:PySys_SetArgvEx
102+
...
103+
}
104+
{
105+
<ignore_py_error0.5>
106+
Memcheck:Cond
107+
...
108+
fun:_PyMethodDef_RawFastCallKeywords
109+
...
110+
}
111+
{
112+
<ignore_py_error0>
113+
Memcheck:Value8
114+
fun:lzf_compress
115+
}
116+
{
117+
<ignore_py_error1>
118+
Memcheck:Addr4
119+
obj:*libpython2.7*
120+
}
121+
{
122+
<ignore_py_error1>
123+
Memcheck:Addr8
124+
obj:*libpython2.7*
125+
}
126+
{
127+
<ignore_py_error2>
128+
Memcheck:Value8
129+
obj:*libpython2.7*
130+
}
131+
{
132+
<ignore_py_error3>
133+
Memcheck:Cond
134+
obj:*libpython2.7*
135+
}
136+
{
137+
<ignore_py_error4>
138+
Memcheck:Leak
139+
...
140+
obj:*libpython2.7*
141+
...
142+
}
143+
{
144+
<ignore_libevent_issues>
145+
Memcheck:Leak
146+
...
147+
fun:redisLibeventAttach
148+
...
149+
}
150+
{
151+
<ignore_compiler_visit_expr_for_now>
152+
Memcheck:Leak
153+
...
154+
fun:compiler_visit_expr
155+
...
156+
}
157+
{
158+
<ignore_compiler_visit_expr_for_now>
159+
Memcheck:Leak
160+
...
161+
fun:redisConnectNonBlock
162+
...
163+
}
164+
{
165+
<ignore_connection_leak_for_now>
166+
Memcheck:Leak
167+
...
168+
fun:_redisContextConnectTcp
169+
...
170+
}
171+
{
172+
Libuv_epoll_ctl
173+
Memcheck:Param
174+
epoll_ctl(event)
175+
fun:epoll_ctl
176+
...
177+
}

‎tests/flow/Install_RedisGears.sh‎

Lines changed: 97 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,97 @@
1+
#!/bin/bash
2+
3+
set -e
4+
5+
HERE="$(cd "$(dirname "${BASH_SOURCE[0]}")" >/dev/null 2>&1 && pwd)"
6+
ROOT=$(cd $HERE/../.. && pwd)
7+
READIES=$ROOT/opt/readies
8+
. $READIES/shibumi/defs
9+
10+
if [[ "$1" == "--help" || "$1" == "help" || "$HELP" == "1" ]]; then
11+
cat <<-END
12+
Obtain RedisGears module binaries
13+
14+
Install_RedisGears.sh [--help|help]
15+
16+
Argument variables:
17+
GEARS_OSNICK=nick Get binaries for give osnick
18+
GEARS_PATH=dir Get binaries from given Gears repo
19+
NOP=1 No operation
20+
HELP=1 Show help
21+
22+
END
23+
exit 0
24+
fi
25+
26+
OP=""
27+
[[ "$NOP" == "1" ]] && OP=echo
28+
29+
os="$($READIES/bin/platform --os)"
30+
arch="$($READIES/bin/platform --arch)"
31+
32+
if [[ ! -z "$GEARS_PATH" ]]; then
33+
platform="$($READIES/bin/platform -t)"
34+
else
35+
if [[ "$os" != "linux" || "$arch" != "x64" ]]; then
36+
eprint "Cannot match binary artifacts - build RedisGears and set GEARS_PATH"
37+
exit 1
38+
fi
39+
40+
dist="$($READIES/bin/platform --dist)"
41+
nick="$($READIES/bin/platform --osnick)"
42+
43+
if [[ $dist == "ubuntu" ]]; then
44+
if [[ $nick != "bionic" && $nick != "xenial" && $nick != "trusty" ]]; then
45+
nick="bionic"
46+
fi
47+
elif [[ $dist == debian ]]; then
48+
nick=bionic
49+
elif [[ $dist == centos || $dist == redhat || $dist == fedora ]]; then
50+
nick=centos7
51+
elif [[ ! -z "$GEARS_OSNICK" ]]; then
52+
nick=$GEARS_OSNICK
53+
else
54+
eprint "Cannot match binary artifacts - build RedisGears and set GEARS_PATH"
55+
exit 1
56+
fi
57+
platform="${os}-${nick}-${arch}"
58+
fi
59+
60+
GEARS_S3_URL="http://redismodules.s3.amazonaws.com/redisgears/snapshots"
61+
GEARS_MOD="redisgears.${platform}.master.zip"
62+
GEARS_DEPS="redisgears-python.${platform}.master.tgz"
63+
64+
FINAL_WORK_DIR="$ROOT/bin/$($READIES/bin/platform -t)/RedisGears"
65+
66+
if [[ -d $FINAL_WORK_DIR && -f $FINAL_WORK_DIR/redisgears.so ]]; then
67+
echo "RedisGears is in ${FINAL_WORK_DIR}"
68+
exit 0
69+
fi
70+
71+
$OP mkdir -p $(dirname $FINAL_WORK_DIR)
72+
$OP rm -rf ${FINAL_WORK_DIR}.*
73+
WORK_DIR=$(mktemp -d ${FINAL_WORK_DIR}.XXXXXX)
74+
$OP mkdir -p $WORK_DIR
75+
76+
if [[ -z $GEARS_PATH ]]; then
77+
F_GEARS_MOD="$WORK_DIR/$GEARS_MOD"
78+
if [[ ! -f $F_GEARS_MOD ]]; then
79+
echo "Download RedisGears ..."
80+
$OP wget -q -P $WORK_DIR $GEARS_S3_URL/$GEARS_MOD
81+
fi
82+
83+
F_GEARS_DEPS="$WORK_DIR/$GEARS_DEPS"
84+
if [[ ! -f $F_GEARS_DEPS ]]; then
85+
echo "Download RedisGears deps ..."
86+
$OP wget -q -P $WORK_DIR $GEARS_S3_URL/$GEARS_DEPS
87+
fi
88+
else
89+
F_GEARS_MOD="${GEARS_PATH}/artifacts/snapshot/${GEARS_MOD}"
90+
F_GEARS_DEPS="${GEARS_PATH}/artifacts/snapshot/${GEARS_DEPS}"
91+
[[ ! -f $F_GEARS_MOD ]] && { eprint "$F_GEARS_MOD is missing"; exit 1; }
92+
[[ ! -f $F_GEARS_DEPS ]] && { eprint "$F_GEARS_DEPS is missing"; exit 1; }
93+
fi
94+
95+
$OP unzip -q $F_GEARS_MOD -d $WORK_DIR
96+
$OP tar --no-same-owner -C $WORK_DIR -xzf $F_GEARS_DEPS
97+
$OP mv $WORK_DIR $FINAL_WORK_DIR

‎tests/flow/includes.py‎

Lines changed: 7 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -10,12 +10,14 @@
1010
from skimage.io import imread
1111
from skimage.transform import resize
1212

13-
try:
14-
sys.path.insert(0, os.path.join(os.path.dirname(__file__), "../../deps/readies"))
15-
import paella
16-
except:
17-
pass
1813

14+
sys.path.insert(0, os.path.join(os.path.dirname(__file__), "../../opt/readies"))
15+
import paella
16+
17+
ROOT = os.environ.get("ROOT", None)
18+
if ROOT is None:
19+
sys.stderr.write("ROOT was not defined in the environment.\n")
20+
sys.exit(1)
1921
MAX_ITERATIONS = 2 if os.environ.get("MAX_ITERATIONS") == None else os.environ.get("MAX_ITERATIONS")
2022
TEST_TF = os.environ.get("TEST_TF") != "0" and os.environ.get("WITH_TF") != "0"
2123
TEST_TFLITE = os.environ.get("TEST_TFLITE") != "0" and os.environ.get("WITH_TFLITE") != "0"

‎tests/flow/tests.sh‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -136,6 +136,7 @@ cd $ROOT/tests/flow
136136

137137
install_git_lfs
138138
check_redis_server
139+
./Install_RedisGears.sh
139140

140141
[[ ! -z $REDIS ]] && RL_TEST_ARGS+=" --env exiting-env --existing-env-addr $REDIS" run_tests "redis-server: $REDIS"
141142
[[ $GEN == 1 ]] && run_tests

0 commit comments

Comments
 (0)