Skip to content
This repository has been archived by the owner on Sep 18, 2024. It is now read-only.

Commit

Permalink
Merge branch 'master' into dev-mac-support
Browse files Browse the repository at this point in the history
  • Loading branch information
@leckie-chn
leckie-chn authored Nov 13, 2018
2 parents a550686 + 221e951 commit cca83a2
Show file tree
Hide file tree
Showing 229 changed files with 15,113 additions and 12,837 deletions.
7 changes: 6 additions & 1 deletion .gitignore
View file
Original file line number Diff line number Diff line change
Expand Up @@ -61,4 +61,9 @@ typings/
.next

# Pycharm Project files
.idea
.idea

# Python cache files
__pycache__
build
*.egg-info
249 changes: 93 additions & 156 deletions Makefile
View file
Original file line number Diff line number Diff line change
@@ -1,6 +1,5 @@
# Setting variables

SHELL := /bin/bash
PIP_INSTALL := python3 -m pip install
PIP_UNINSTALL := python3 -m pip uninstall

Expand All @@ -16,6 +15,7 @@ else
$(error platform $(UNAME_S) not supported)
endif


## Colorful output
_INFO := $(shell echo -e '$(ESC_CMD)[1;36m')
_WARNING := $(shell echo -e '$(ESC_CMD)[1;33m')
Expand All @@ -24,14 +24,10 @@ _END := $(shell echo -e '$(ESC_CMD)[0m')
## Install directories
ifeq ($(shell id -u), 0) # is root
_ROOT := 1
BIN_PATH ?= /usr/bin
INSTALL_PREFIX ?= /usr/share
EXAMPLES_PATH ?= $(NNI_INSTALL_PATH)/examples
BASH_COMP_PREFIX ?= /usr/share/bash-completion/completions
ROOT_FOLDER ?= $(shell python3 -c 'import site; from pathlib import Path; print(Path(site.getsitepackages()[0]).parents[2])')
BASH_COMP_SCRIPT ?= /usr/share/bash-completion/completions/nnictl
else # is normal user
BIN_PATH ?= ${HOME}/.local/bin
INSTALL_PREFIX ?= ${HOME}/.local
EXAMPLES_PATH ?= ${HOME}/nni/examples
ROOT_FOLDER ?= $(shell python3 -c 'import site; from pathlib import Path; print(Path(site.getusersitepackages()).parents[2])')
ifndef VIRTUAL_ENV
PIP_MODE ?= --user
endif
Expand All @@ -42,80 +38,30 @@ BASH_COMP_SCRIPT := $(BASH_COMP_PREFIX)/nnictl
NNI_INSTALL_PATH ?= $(INSTALL_PREFIX)/nni
NNI_TMP_PATH ?= /tmp

## Dependency information
NNI_NODE_VERSION ?= v10.12.0
NNI_NODE_TARBALL ?= node-$(NNI_NODE_VERSION)-$(OS_SPEC)-x64.tar.xz
NNI_NODE_PATH ?= $(NNI_INSTALL_PATH)/node

NNI_YARN_VERSION ?= v1.10.1
NNI_YARN_TARBALL ?= yarn-$(NNI_YARN_VERSION).tar.gz
NNI_YARN_PATH ?= $(NNI_TMP_PATH)/nni-yarn

## Check if dependencies have been installed globally
ifeq (, $(shell command -v node 2>/dev/null))
$(info $(_INFO) Node.js not found $(_END))
_MISS_DEPS := 1 # node not found
else
_VER := $(shell node --version)
_NEWER := $(shell echo -e "$(NNI_NODE_VERSION)\n$(_VER)" | sort -Vr | head -n 1)
ifneq ($(_VER), $(_NEWER))
$(info $(_INFO) Node.js version not match $(_END))
_MISS_DEPS := 1 # node outdated
endif
endif
ifeq (, $(shell command -v yarnpkg 2>/dev/null))
$(info $(_INFO) Yarn not found $(_END))
_MISS_DEPS := 1 # yarn not found
endif

ifdef _MISS_DEPS
$(info $(_INFO) Missing dependencies, use local toolchain $(_END))
NNI_NODE := $(NNI_NODE_PATH)/bin/node
NNI_YARN := PATH=$(NNI_NODE_PATH)/bin:$${PATH} $(NNI_YARN_PATH)/bin/yarn
else
$(info $(_INFO) All dependencies found, use global toolchain $(_END))
NNI_NODE := node
NNI_YARN := yarnpkg
endif


# Setting variables end
BIN_FOLDER ?= $(ROOT_FOLDER)/bin
NNI_PKG_FOLDER ?= $(ROOT_FOLDER)/nni

## Dependency information
NNI_NODE_TARBALL ?= /tmp/nni-node-linux-x64.tar.xz
NNI_NODE_FOLDER = /tmp/nni-node-linux-x64
NNI_NODE ?= $(BIN_FOLDER)/node
NNI_YARN_TARBALL ?= /tmp/nni-yarn.tar.gz
NNI_YARN_FOLDER ?= /tmp/nni-yarn
NNI_YARN := PATH=$(BIN_FOLDER):$${PATH} $(NNI_YARN_FOLDER)/bin/yarn

# Main targets

.PHONY: build
build:
#$(_INFO) Building NNI Manager $(_END)
cd src/nni_manager && $(NNI_YARN) && $(NNI_YARN) build

#$(_INFO) Building WebUI $(_END)
cd src/webui && $(NNI_YARN) && $(NNI_YARN) build

#$(_INFO) Building Python SDK $(_END)
cd src/sdk/pynni && python3 setup.py build

#$(_INFO) Building nnictl $(_END)
cd tools && python3 setup.py build

# Standard installation target
# Must be invoked after building
.PHONY: install
install: install-python-modules
install: install-node-modules
install: install-scripts
install: install-examples
install:
#$(_INFO) Complete! You may want to add $(BIN_PATH) to your PATH environment $(_END)


# Target for remote machine workers
# Only installs core SDK module
.PHONY: remote-machine-install
remote-machine-install:
cd src/sdk/pynni && python3 setup.py install $(PIP_MODE)


# All-in-one target for non-expert users
# Installs NNI as well as its dependencies, and update bashrc to set PATH
.PHONY: easy-install
Expand All @@ -124,10 +70,37 @@ easy-install: install-dependencies
easy-install: build
easy-install: install
easy-install: update-bash-config

easy-install:
#$(_INFO) Complete! $(_END)

# All-in-one target for developer users
# Install NNI as well as its dependencies, and update bashrc to set PATH
.PHONY: dev-easy-install
dev-easy-install: dev-check-perm
dev-easy-install: install-dependencies
dev-easy-install: build
dev-easy-install: dev-install
dev-easy-install: update-bash-config
dev-easy-install:
#$(_INFO) Complete! $(_END)

# Standard installation target
# Must be invoked after building
.PHONY: install
install: install-python-modules
install: install-node-modules
install: install-scripts
install:
#$(_INFO) Complete! You may want to add $(BIN_FOLDER) to your PATH environment $(_END)

# Target for NNI developers
# Creates symlinks instead of copying files
.PHONY: dev-install
dev-install: dev-install-python-modules
dev-install: dev-install-node-modules
dev-install: install-scripts
dev-install:
#$(_INFO) Complete! You may want to add $(BIN_FOLDER) to your PATH environment $(_END)

# Target for setup.py
# Do not invoke this manually
Expand All @@ -136,58 +109,54 @@ pip-install: install-dependencies
pip-install: build
pip-install: install-node-modules
pip-install: install-scripts
pip-install: install-examples
pip-install: update-bash-config


# Target for NNI developers
# Creates symlinks instead of copying files
.PHONY: dev-install
dev-install: check-dev-env
dev-install: install-dev-modules
dev-install: install-scripts
dev-install:
#$(_INFO) Complete! You may want to add $(BIN_PATH) to your PATH environment $(_END)


.PHONY: uninstall
uninstall:
-$(PIP_UNINSTALL) -y nni
-$(PIP_UNINSTALL) -y nnictl
-rm -rf $(NNI_INSTALL_PATH)
-rm -f $(BIN_PATH)/nnimanager
-rm -f $(BIN_PATH)/nnictl
-rm -rf $(NNI_PKG_FOLDER)
-rm -f $(BIN_FOLDER)/node
-rm -f $(BIN_FOLDER)/nnictl
-rm -f $(BASH_COMP_SCRIPT)
-[ $(EXAMPLES_PATH) = ${PWD}/examples ] || rm -rf $(EXAMPLES_PATH)

# Main targets end
.PHONY: clean
clean:
-rm -rf tools/build
-rm -rf tools/nnictl.egg-info
-rm -rf src/nni_manager/dist
-rm -rf src/nni_manager/node_modules
-rm -rf src/sdk/pynni/build
-rm -rf src/sdk/pynni/nni_sdk.egg-info
-rm -rf src/webui/build
-rm -rf src/webui/node_modules

# Main targets end

# Helper targets

$(NNI_NODE_TARBALL):
#$(_INFO) Downloading Node.js $(_END)
wget https://nodejs.org/dist/$(NNI_NODE_VERSION)/$(NNI_NODE_TARBALL)
wget https://aka.ms/nodejs-download -O $(NNI_NODE_TARBALL)

$(NNI_YARN_TARBALL):
#$(_INFO) Downloading Yarn $(_END)
wget https://github.com/yarnpkg/yarn/releases/download/$(NNI_YARN_VERSION)/$(NNI_YARN_TARBALL)
wget https://aka.ms/yarn-download -O $(NNI_YARN_TARBALL)

.PHONY: intall-dependencies
.PHONY: install-dependencies
install-dependencies: $(NNI_NODE_TARBALL) $(NNI_YARN_TARBALL)
#$(_INFO) Cleaning $(_END)
rm -rf $(NNI_NODE_PATH)
rm -rf $(NNI_YARN_PATH)
if [ ! -d $(NNI_INSTALL_PATH) ]; then mkdir -p $(NNI_INSTALL_PATH); fi
if [ ! -d $(NNI_TMP_PATH) ]; then mkdir -p $(NNI_TMP_PATH); fi

#$(_INFO) Extracting Node.js $(_END)
tar -xf $(NNI_NODE_TARBALL)
mv -f node-$(NNI_NODE_VERSION)-$(OS_SPEC)-x64 $(NNI_NODE_PATH)
rm -rf $(NNI_NODE_FOLDER)
mkdir $(NNI_NODE_FOLDER)
tar -xf $(NNI_NODE_TARBALL) -C $(NNI_NODE_FOLDER) --strip-components 1
mkdir -p $(BIN_FOLDER)
rm -f $(NNI_NODE)
cp $(NNI_NODE_FOLDER)/bin/node $(NNI_NODE)

#$(_INFO) Extracting Yarn $(_END)
tar -xf $(NNI_YARN_TARBALL)
mv -f yarn-$(NNI_YARN_VERSION) $(NNI_YARN_PATH)
rm -rf $(NNI_YARN_FOLDER)
mkdir $(NNI_YARN_FOLDER)
tar -xf $(NNI_YARN_TARBALL) -C $(NNI_YARN_FOLDER) --strip-components 1

.PHONY: install-python-modules
install-python-modules:
Expand All @@ -197,69 +166,42 @@ install-python-modules:
#$(_INFO) Installing nnictl $(_END)
cd tools && $(PIP_INSTALL) $(PIP_MODE) .

.PHONY: install-node-modules
install-node-modules:
mkdir -p $(NNI_INSTALL_PATH)
rm -rf src/nni_manager/dist/node_modules
rm -rf $(NNI_INSTALL_PATH)/nni_manager

#$(_INFO) Installing NNI Manager $(_END)
cp -r src/nni_manager/dist $(NNI_INSTALL_PATH)/nni_manager
cp -r src/nni_manager/node_modules $(NNI_INSTALL_PATH)/nni_manager/node_modules

#$(_INFO) Installing WebUI $(_END)
cp -r src/webui/build $(NNI_INSTALL_PATH)/nni_manager/static


.PHONY: install-dev-modules
install-dev-modules:
.PHONY: dev-install-python-modules
dev-install-python-modules:
#$(_INFO) Installing Python SDK $(_END)
cd src/sdk/pynni && $(PIP_INSTALL) $(PIP_MODE) -e .

#$(_INFO) Installing nnictl $(_END)
cd tools && $(PIP_INSTALL) $(PIP_MODE) -e .

mkdir -p $(NNI_INSTALL_PATH)

#$(_INFO) Installing NNI Manager $(_END)
ln -sf ${PWD}/src/nni_manager/dist $(NNI_INSTALL_PATH)/nni_manager
ln -sf ${PWD}/src/nni_manager/node_modules $(NNI_INSTALL_PATH)/nni_manager/node_modules

#$(_INFO) Installing WebUI $(_END)
ln -sf ${PWD}/src/webui/build $(NNI_INSTALL_PATH)/nni_manager/static

.PHONY: install-node-modules
install-node-modules:
#$(_INFO) Installing NNI Package $(_END)
rm -rf $(NNI_PKG_FOLDER)
cp -r src/nni_manager/dist $(NNI_PKG_FOLDER)
cp src/nni_manager/package.json $(NNI_PKG_FOLDER)
$(NNI_YARN) --prod --cwd $(NNI_PKG_FOLDER)
cp -r src/webui/build $(NNI_PKG_FOLDER)/static

.PHONY: dev-install-node-modules
dev-install-node-modules:
#$(_INFO) Installing NNI Package $(_END)
rm -rf $(NNI_PKG_FOLDER)
ln -sf ${PWD}/src/nni_manager/dist $(NNI_PKG_FOLDER)
ln -sf ${PWD}/src/nni_manager/node_modules $(NNI_PKG_FOLDER)/node_modules
ln -sf ${PWD}/src/webui/build $(NNI_PKG_FOLDER)/static

.PHONY: install-scripts
install-scripts:
mkdir -p $(BIN_PATH)

echo '#!/bin/sh' > $(BIN_PATH)/nnimanager
echo 'cd $(NNI_INSTALL_PATH)/nni_manager' >> $(BIN_PATH)/nnimanager
echo '$(NNI_NODE) main.js $$@' >> $(BIN_PATH)/nnimanager
chmod +x $(BIN_PATH)/nnimanager

echo '#!/bin/sh' > $(BIN_PATH)/nnictl
echo 'NNI_MANAGER=$(BIN_PATH)/nnimanager \' >> $(BIN_PATH)/nnictl
echo 'python3 -m nnicmd.nnictl $$@' >> $(BIN_PATH)/nnictl
chmod +x $(BIN_PATH)/nnictl

if [ ! -d $(BASH_COMP_PREFIX) ]; then mkdir -p $(BASH_COMP_PREFIX); fi
install -m644 tools/bash-completion $(BASH_COMP_SCRIPT)


.PHONY: install-examples
install-examples:
mkdir -p $(EXAMPLES_PATH)
[ $(EXAMPLES_PATH) = ${PWD}/examples ] || cp -r examples/* $(EXAMPLES_PATH)

install -Dm644 tools/bash-completion $(BASH_COMP_SCRIPT)

.PHONY: update-bash-config
ifndef _ROOT
update-bash-config:
#$(_INFO) Updating bash configurations $(_END)
ifeq (, $(shell echo $$PATH | tr ':' '\n' | grep -x '$(BIN_PATH)')) # $(BIN_PATH) not in PATH
#$(_WARNING) NOTE: adding $(BIN_PATH) to PATH in bashrc $(_END)
echo 'export PATH="$$PATH:$(BIN_PATH)"' >> ~/.bashrc
ifeq (, $(shell echo $$PATH | tr ':' '\n' | grep -x '$(BIN_FOLDER)')) # $(BIN_FOLDER) not in PATH
#$(_WARNING) NOTE: adding $(BIN_FOLDER) to PATH in bashrc $(_END)
echo 'export PATH="$$PATH:$(BIN_FOLDER)"' >> ~/.bashrc
endif
ifeq (, $(shell (source ~/.bash_completion ; command -v _nnictl) 2>/dev/null)) # completion not installed
#$(_WARNING) NOTE: adding $(BASH_COMP_SCRIPT) to ~/.bash_completion $(_END)
Expand All @@ -269,7 +211,6 @@ else
update-bash-config: ;
endif


.PHONY: check-perm
ifdef _ROOT
check-perm:
Expand All @@ -281,16 +222,12 @@ else
check-perm: ;
endif


.PHONY: check-dev-env
check-dev-env:
#$(_INFO) Checking developing environment... $(_END)
.PHONY: dev-check-perm
ifdef _ROOT
dev-check-perm:
$(error You should not develop NNI as root)
else
dev-check-perm: ;
endif
ifdef _MISS_DEPS
# $(error Please install Node.js and Yarn to develop NNI)
endif
#$(_INFO) Pass! $(_END)

# Helper targets end
Loading

0 comments on commit cca83a2

Please sign in to comment.