diff --git a/MAINTAINERS.md b/MAINTAINERS.md index cb19c8501..66d3ab9dd 100644 --- a/MAINTAINERS.md +++ b/MAINTAINERS.md @@ -5,6 +5,12 @@ | Baohua Yang | yeasy | baohua | yangbaohua@gmail.com | | Haitao Yue | hightall | hightall | hightallyht@gmail.com | | Tong Li | tongli | tongli | litong01@us.ibm.com | +| Qiang Xu | XuHugo | XuHugo | xq-310@163.com | + +## Retired Maintainers + +| Name | GitHub | RocketChat | Email | +|---|---|---|---| | Luke Chen | LordGoodman | luke_chen | jiahaochen1993@gmail.com | Creative Commons License
This work is licensed under a Creative Commons Attribution 4.0 International License. diff --git a/build_image/docker/cello-hlf/Dockerfile b/build_image/docker/cello-hlf/Dockerfile index 641bedcde..248a286a9 100644 --- a/build_image/docker/cello-hlf/Dockerfile +++ b/build_image/docker/cello-hlf/Dockerfile @@ -36,7 +36,7 @@ ENV BASE_VERSION=2.2.0 ENV PROJECT_VERSION=2.2.0 ENV HLF_CA_VERSION=1.4.8 -# generic environment (core.yaml) for builder and runtime: builder: $(DOCKER_NS)/fabric-ccenv:$(TWO_DIGIT_VERSION) +# generic environment (core.yaml) for builder and runtime: e.g., builder: $(DOCKER_NS)/fabric-ccenv:$(TWO_DIGIT_VERSION), golang, java, node ENV DOCKER_NS=hyperledger ENV TWO_DIGIT_VERSION=2.2 @@ -80,12 +80,12 @@ RUN mkdir -p /var/hyperledger/production \ # Install development dependencies RUN apt-get update \ - && apt-get install -y apt-utils python-dev \ + && apt-get install -y apt-utils python3-dev \ && apt-get install -y libsnappy-dev zlib1g-dev libbz2-dev libyaml-dev libltdl-dev libtool \ - && apt-get install -y python-pip \ + && apt-get install -y python3-pip \ && apt-get install -y vim tree jq unzip \ - && pip install behave nose docker-compose \ - && pip install pyinotify \ + && pip3 install behave nose docker-compose \ + && pip3 install pyinotify \ && rm -rf /var/cache/apt # Install yq to update config for fabric-ca @@ -99,9 +99,8 @@ RUN go get github.com/golang/protobuf/protoc-gen-go \ && go get github.com/AlekSi/gocov-xml \ && go get golang.org/x/tools/cmd/goimports \ && go get golang.org/x/lint/golint \ - && go get github.com/estesp/manifest-tool \ + && go get github.com/estesp/manifest-tool/... \ && go get github.com/client9/misspell/cmd/misspell \ - && go get github.com/estesp/manifest-tool \ && go get github.com/onsi/ginkgo/ginkgo # Clone the Hyperledger Fabric code and cp sample config files diff --git a/build_image/docker/cello-hlf/scripts/init.sh b/build_image/docker/cello-hlf/scripts/init.sh index 09b739e41..aabc3d551 100644 --- a/build_image/docker/cello-hlf/scripts/init.sh +++ b/build_image/docker/cello-hlf/scripts/init.sh @@ -36,6 +36,14 @@ cmd=$1 # The path to store the files cfg_path=${FABRIC_CFG_PATH:-/etc/hyperledger/fabric} +# Clean the potential existing msp and tls paths first +if [ ! -z "${HLF_NODE_MSP}" ] && [ -d ${cfg_path}/msp ]; then + mv ${cfg_path}/msp ${cfg_path}/msp.bak +fi +if [ ! -z "${HLF_NODE_TLS}" ] && [ -d ${cfg_path}/tls ]; then + mv ${cfg_path}/tls ${cfg_path}/tls.bak +fi + # Read each file from env and store under the ${cfg_path} for name in HLF_NODE_MSP \ HLF_NODE_TLS \ diff --git a/src/api-engine/api/config.py b/src/api-engine/api/config.py new file mode 100644 index 000000000..be798ed7c --- /dev/null +++ b/src/api-engine/api/config.py @@ -0,0 +1,9 @@ +# +# SPDX-License-Identifier: Apache-2.0 +# +CELLO_HOME = "/opt/cello" +FABRIC_TOOL = "/opt/bin" + +FABRIC_PEER_CFG = "/opt/node/peer.yaml.bak" +FABRIC_ORDERER_CFG = "/opt/node/orderer.yaml.bak" +FABRIC_CA_CFG = "/opt/node/ca.yaml.bak" \ No newline at end of file diff --git a/src/api-engine/api/models.py b/src/api-engine/api/models.py index e4525dce8..2e3c01011 100644 --- a/src/api-engine/api/models.py +++ b/src/api-engine/api/models.py @@ -15,6 +15,7 @@ from django.db import models from django.dispatch import receiver from django.db.models.signals import post_save +from django.contrib.postgres.fields import ArrayField from api.common.enums import ( HostStatus, @@ -71,6 +72,30 @@ class Organization(models.Model): default="", max_length=64, help_text="Name of organization" ) created_at = models.DateTimeField(auto_now_add=True) + msp = models.TextField(help_text="msp of organization", null=True) + tls = models.TextField(help_text="tls of organization", null=True) + agents = models.CharField( + help_text="agent of organization", + max_length=128, + default="", + ) + network = models.ForeignKey( + "Network", + help_text="Network to which the organization belongs", + null=True, + related_name="network", + on_delete=models.SET_NULL + ) + channel = models.ForeignKey( + "Channel", + help_text="channel to which the organization belongs", + null=True, + related_name="channel", + on_delete=models.SET_NULL + ) + + class Meta: + ordering = ("-created_at",) class UserProfile(AbstractUser): @@ -137,21 +162,17 @@ class Agent(models.Model): max_length=64, default=random_name("agent"), ) - image = models.CharField( - help_text="Image name for deploy agent", max_length=64, default="" - ) - ip = models.GenericIPAddressField(help_text="Agent IP Address") - govern = models.ForeignKey( - Govern, - help_text="Govern of agent", + urls = models.URLField( + help_text="Agent URL", null=True, - on_delete=models.CASCADE, + blank=True ) organization = models.ForeignKey( "Organization", null=True, on_delete=models.CASCADE, help_text="Organization of agent", + related_name="organization", ) status = models.CharField( help_text="Status of agent", @@ -159,34 +180,12 @@ class Agent(models.Model): max_length=10, default=HostStatus.Active.name.lower(), ) - log_level = models.CharField( - help_text="Log level of agent", - choices=LogLevel.to_choices(True), - max_length=10, - default=LogLevel.Info.name.lower(), - ) type = models.CharField( help_text="Type of agent", choices=HostType.to_choices(True), max_length=32, default=HostType.Docker.name.lower(), ) - schedulable = models.BooleanField( - help_text="Whether agent can be scheduled", default=True - ) - capacity = models.IntegerField( - help_text="Capacity of agent", - default=1, - validators=[MinValueValidator(1), MaxValueValidator(MAX_CAPACITY)], - ) - node_capacity = models.IntegerField( - help_text="Capacity of node", - default=6, - validators=[ - MinValueValidator(1), - MaxValueValidator(MAX_NODE_CAPACITY), - ], - ) config_file = models.FileField( help_text="Config file for agent", max_length=256, @@ -284,8 +283,10 @@ class Network(models.Model): default=make_uuid, editable=True, ) - govern = models.ForeignKey( - Govern, help_text="Govern of node", null=True, on_delete=models.CASCADE + name = models.CharField( + help_text="network name, can be generated automatically.", + max_length=64, + default=random_name("netowrk"), ) type = models.CharField( help_text="Type of network, %s" % NetworkType.values(), @@ -304,6 +305,12 @@ class Network(models.Model): created_at = models.DateTimeField( help_text="Create time of network", auto_now_add=True ) + consensus = models.CharField( + help_text="Consensus of network", max_length=128, default="raft", + ) + organizations = ArrayField( + models.CharField(max_length=128, blank=True), help_text="organizations of network", default=list, null=True + ) class Meta: ordering = ("-created_at",) @@ -449,21 +456,6 @@ class Node(models.Model): editable=True, ) name = models.CharField(help_text="Node name", max_length=64, default="") - network_type = models.CharField( - help_text="Network type of node", - choices=NetworkType.to_choices(), - default=NetworkType.Fabric.value, - max_length=64, - ) - network_version = models.CharField( - help_text=""" - Version of network for node. - Fabric supported versions: %s - """ - % (FabricVersions.values()), - max_length=64, - default="", - ) type = models.CharField( help_text=""" Node type defined for network. @@ -472,18 +464,6 @@ class Node(models.Model): % (FabricNodeType.names()), max_length=64, ) - ca = models.ForeignKey( - FabricCA, - help_text="CA configuration of node", - null=True, - on_delete=models.CASCADE, - ) - peer = models.ForeignKey( - FabricPeer, - help_text="Peer configuration of node", - null=True, - on_delete=models.CASCADE, - ) urls = JSONField( help_text="URL configurations for node", null=True, @@ -496,24 +476,26 @@ class Node(models.Model): null=True, on_delete=models.CASCADE, ) - govern = models.ForeignKey( - Govern, help_text="Govern of node", null=True, on_delete=models.CASCADE - ) - organization = models.ForeignKey( + org = models.ForeignKey( Organization, help_text="Organization of node", null=True, + related_name="org", on_delete=models.CASCADE, ) - agent = models.ForeignKey( - Agent, help_text="Agent of node", null=True, on_delete=models.CASCADE - ) - network = models.ForeignKey( - Network, - help_text="Network which node joined.", - on_delete=models.CASCADE, - null=True, - ) + # agent = models.ForeignKey( + # Agent, + # help_text="Agent of node", + # null=True, + # related_name="network", + # on_delete=models.CASCADE + # ) + # network = models.ForeignKey( + # Network, + # help_text="Network which node joined.", + # on_delete=models.CASCADE, + # null=True, + # ) created_at = models.DateTimeField( help_text="Create time of network", auto_now_add=True ) @@ -523,18 +505,18 @@ class Node(models.Model): max_length=64, default=NodeStatus.Deploying.name.lower(), ) - compose_file = models.FileField( - help_text="Compose file for node, if agent type is docker.", + config_file = models.TextField( + help_text="Config file of node", max_length=256, - upload_to=get_compose_file_path, blank=True, null=True, ) - file = models.FileField( - help_text="File of node", - max_length=256, - blank=True, - upload_to=get_node_file_path, + msp = models.TextField( + help_text="msp of node", + null=True, + ) + tls = models.TextField( + help_text="tls of node", null=True, ) @@ -561,22 +543,22 @@ def save( force_insert, force_update, using, update_fields ) - def delete(self, using=None, keep_parents=False): - if self.compose_file: - compose_file_path = Path(self.compose_file.path) - if os.path.isdir(os.path.dirname(compose_file_path)): - shutil.rmtree(os.path.dirname(compose_file_path)) - - # remove related files of node - if self.file: - file_path = Path(self.file.path) - if os.path.isdir(os.path.dirname(file_path)): - shutil.rmtree(os.path.dirname(file_path)) - - if self.ca: - self.ca.delete() - - super(Node, self).delete(using, keep_parents) + # def delete(self, using=None, keep_parents=False): + # if self.compose_file: + # compose_file_path = Path(self.compose_file.path) + # if os.path.isdir(os.path.dirname(compose_file_path)): + # shutil.rmtree(os.path.dirname(compose_file_path)) + # + # # remove related files of node + # if self.file: + # file_path = Path(self.file.path) + # if os.path.isdir(os.path.dirname(file_path)): + # shutil.rmtree(os.path.dirname(file_path)) + # + # if self.ca: + # self.ca.delete() + # + # super(Node, self).delete(using, keep_parents) class NodeUser(models.Model): @@ -684,3 +666,85 @@ class File(models.Model): class Meta: ordering = ("-created_at",) + + class User(models.Model): + id = models.UUIDField( + primary_key=True, + help_text="ID of user", + default=make_uuid, + editable=True, + ) + name = models.CharField( + help_text="user name", max_length=128 + ) + roles = models.CharField( + help_text="roles of user", max_length=128 + ) + organization = models.ForeignKey("Organization", on_delete=models.CASCADE) + attributes = models.CharField( + help_text="attributes of user", max_length=128 + ) + revoked = models.CharField( + help_text="revoked of user", max_length=128 + ) + create_ts = models.DateTimeField( + help_text="Create time of user", auto_now_add=True + ) + msp = models.TextField( + help_text="msp of user", + null=True, + ) + tls = models.TextField( + help_text="tls of user", + null=True, + ) + + class Channel(models.Model): + id = models.UUIDField( + primary_key=True, + help_text="ID of Channel", + default=make_uuid, + editable=False, + unique=True) + name = models.CharField( + help_text="name of channel", max_length=128 + ) + network = models.ForeignKey( + "Network", on_delete=models.CASCADE + ) + create_ts = models.DateTimeField( + help_text="Create time of Channel", auto_now_add=True + ) + + class ChainCode(models.Model): + id = models.UUIDField( + primary_key=True, + help_text="ID of chainCode", + default=make_uuid, + editable=False, + unique=True + ) + name = models.CharField( + help_text="ChainCode name", max_length=128 + ) + version = models.CharField( + help_text="version of chainCode", max_length=128 + ) + creator = models.CharField( + help_text="creator of chainCode", max_length=128 + ) + language = models.CharField( + help_text="language of chainCode", max_length=128 + ) + channel = models.ManyToManyField("Channel") + install_times = models.DateTimeField( + help_text="Create time of install", auto_now_add=True + ) + instantiate_times = models.DateTimeField( + help_text="Create time of instantiate", auto_now_add=True + ) + node = models.ManyToManyField("Node", related_name='node') + status = models.CharField( + help_text="status of chainCode", max_length=128 + ) + diff --git a/src/api-engine/api/routes/agent/serializers.py b/src/api-engine/api/routes/agent/serializers.py index cf3bd3e5b..050963d8b 100644 --- a/src/api-engine/api/routes/agent/serializers.py +++ b/src/api-engine/api/routes/agent/serializers.py @@ -109,6 +109,8 @@ def validate(self, attrs): class AgentCreateBody(serializers.ModelSerializer): + organization = serializers.UUIDField(help_text=IDHelpText) + def to_form_paras(self): custom_paras = to_form_paras(self) @@ -118,21 +120,16 @@ class Meta: model = Agent fields = ( "name", - "capacity", - "node_capacity", - "log_level", "type", - "schedulable", - "ip", - "image", + "urls", "config_file", + "organization", ) extra_kwargs = { - "capacity": {"required": True}, - "node_capacity": {"required": True}, "type": {"required": True}, - "ip": {"required": True}, - "image": {"required": True}, + "urls": {"required": True}, + "name": {"required": True}, + "organization": {"required": False}, "config_file": { "required": False, "validators": [ @@ -145,13 +142,7 @@ class Meta: } def validate(self, attrs): - capacity = attrs.get("capacity") - node_capacity = attrs.get("node_capacity") - if node_capacity < capacity: - raise serializers.ValidationError( - "Node capacity must larger than capacity" - ) - + pass return attrs @@ -187,42 +178,29 @@ class AgentUpdateBody(AgentPatchBody): class AgentResponseSerializer(AgentIDSerializer, serializers.ModelSerializer): - organization_id = serializers.UUIDField( + organization = serializers.UUIDField( help_text="Organization ID", required=False, allow_null=True ) - config_file = serializers.URLField( - help_text="Config file of agent", required=False - ) class Meta: model = Agent fields = ( "id", "name", - "capacity", - "node_capacity", "status", "created_at", - "log_level", "type", - "schedulable", - "organization_id", - "config_file", - "ip", - "image", + "urls", + "organization", ) extra_kwargs = { "id": {"required": True}, "name": {"required": True}, "status": {"required": True}, - "capacity": {"required": True}, - "node_capacity": {"required": True}, "created_at": {"required": True, "read_only": False}, "type": {"required": True}, - "log_level": {"required": True}, - "schedulable": {"required": True}, - "ip": {"required": True}, - "image": {"required": True}, + "organization": {"required": True}, + "urls": {"required": True}, } diff --git a/src/api-engine/api/routes/agent/views.py b/src/api-engine/api/routes/agent/views.py index 6f2739d14..f5fdc98aa 100644 --- a/src/api-engine/api/routes/agent/views.py +++ b/src/api-engine/api/routes/agent/views.py @@ -22,7 +22,7 @@ NoResource, ResourceInUse, ) -from api.models import Agent, KubernetesConfig +from api.models import Agent, KubernetesConfig, Organization from api.routes.agent.serializers import ( AgentQuery, AgentListResponse, @@ -39,6 +39,7 @@ class AgentViewSet(viewsets.ViewSet): + """Class represents agent related operations.""" authentication_classes = (JSONWebTokenAuthentication,) def get_permissions(self): @@ -62,7 +63,9 @@ def list(self, request): """ List Agents - Filter agents with query parameters. + :param request: query parameter + :return: agent list + :rtype: list """ serializer = AgentQuery(data=request.GET) if serializer.is_valid(raise_exception=True): @@ -74,18 +77,18 @@ def list(self, request): organization = serializer.validated_data.get("organization") query_filters = {} - if organization: - if not request.user.is_operator: - raise PermissionDenied() - query_filters.update({"organization": organization}) - else: - org_name = ( - request.user.organization.name - if request.user.organization - else "" - ) - if request.user.is_administrator: - query_filters.update({"organization__name": org_name}) + # if organization: + # if not request.user.is_operator: + # raise PermissionDenied() + # query_filters.update({"organization": organization}) + # else: + # org_name = ( + # request.user.organization.name + # if request.user.organization + # else "" + # ) + # if request.user.is_administrator: + # query_filters.update({"organization__name": org_name}) if name: query_filters.update({"name__icontains": name}) if agent_status: @@ -100,13 +103,6 @@ def list(self, request): agent_list = [] for agent in agents: agent_dict = agent.__dict__ - agent_dict.update( - { - "config_file": request.build_absolute_uri( - agent.config_file.url - ) - } - ) agent_list.append(agent_dict) response = AgentListResponse( @@ -127,45 +123,41 @@ def create(self, request): """ Create Agent - Create new agent + :param request: create parameter + :return: agent ID + :rtype: uuid """ serializer = AgentCreateBody(data=request.data) if serializer.is_valid(raise_exception=True): name = serializer.validated_data.get("name") - capacity = serializer.validated_data.get("capacity") - node_capacity = serializer.validated_data.get("node_capacity") - log_level = serializer.validated_data.get("log_level") agent_type = serializer.validated_data.get("type") - schedulable = serializer.validated_data.get("schedulable") - parameters = serializer.validated_data.get("parameters") - ip = serializer.validated_data.get("ip") - image = serializer.validated_data.get("image") + urls = serializer.validated_data.get("urls") config_file = serializer.validated_data.get("config_file") + organization = serializer.validated_data.get("organization") body = { - "capacity": capacity, - "node_capacity": node_capacity, "type": agent_type, - "ip": ip, - "image": image, + "urls": urls, + "name": name, } if name: agent_count = Agent.objects.filter( - name=name, organization=request.user.organization + name=name ).count() if agent_count > 0: raise ResourceExists( detail="Name %s already exists" % name ) body.update({"name": name}) - if schedulable is not None: - body.update({"schedulable": schedulable}) - if log_level is not None: - body.update({"log_level": log_level}) - if parameters is not None: - body.update({"parameters": parameters}) + if config_file is not None: body.update({"config_file": config_file}) + if organization is not None: + org = Organization.objects.get(id=organization) + if org.organization.all(): + raise ResourceExists + else: + body.update({"organization": org}) agent = Agent(**body) agent.save() @@ -185,7 +177,10 @@ def retrieve(self, request, pk=None): """ Retrieve agent - Retrieve agent + :param request: destory parameter + :param pk: primary key + :return: none + :rtype: rest_framework.status """ try: if request.user.is_operator: diff --git a/src/api-engine/api/routes/network/serializers.py b/src/api-engine/api/routes/network/serializers.py index 03cd6df4e..7c70258ba 100644 --- a/src/api-engine/api/routes/network/serializers.py +++ b/src/api-engine/api/routes/network/serializers.py @@ -31,6 +31,10 @@ class NetworkQuery(serializers.Serializer): help_text=NetworkStatus.get_info("Network Status:", list_str=True), choices=NetworkStatus.to_choices(True), ) + class Meta: + model = Network + fields = ("page", "per_page", "name") + extra_kwargs = {"name": {"required": False}} class NetworkIDSerializer(serializers.Serializer): @@ -38,12 +42,18 @@ class NetworkIDSerializer(serializers.Serializer): class NetworkResponse(NetworkIDSerializer): - status = serializers.ChoiceField( - help_text=NetworkStatus.get_info("Network Status:", list_str=True), - choices=NetworkStatus.to_choices(True), - ) + id = serializers.UUIDField(help_text="ID of Network") + created_at = serializers.DateTimeField(help_text="Network create time") - updated_at = serializers.DateTimeField(help_text="Network update time") + + class Meta: + model = Network + fields = ("id", "name", "created_at") + extra_kwargs = { + "name": {"required": True}, + "created_at": {"required": True, "read_only": False}, + "id": {"required": True, "read_only": False}, + } class NetworkMemberSerializer(serializers.Serializer): @@ -56,20 +66,13 @@ class NetworkMemberSerializer(serializers.Serializer): class NetworkCreateBody(serializers.ModelSerializer): - method = serializers.ChoiceField( - help_text=NetworkCreateType.get_info( - "Network Create Types:", list_str=True - ), - choices=NetworkCreateType.to_choices(True), - ) class Meta: model = Network - fields = ("type", "version", "method") - extra_kwargs = { - "type": {"required": True}, - "version": {"required": True}, - } + fields = ("name", "consensus", "organizations") + extra_kwargs = {"name": {"required": True}, + "consensus": {"required": True}, + "organizations": {"required": True}} class NetworkMemberResponse(serializers.Serializer): diff --git a/src/api-engine/api/routes/network/views.py b/src/api-engine/api/routes/network/views.py index edcad9310..b26aca2f0 100644 --- a/src/api-engine/api/routes/network/views.py +++ b/src/api-engine/api/routes/network/views.py @@ -7,6 +7,8 @@ from rest_framework.decorators import action from rest_framework.response import Response from drf_yasg.utils import swagger_auto_schema +from django.core.paginator import Paginator +from django.core.exceptions import ObjectDoesNotExist from api.routes.network.serializers import ( NetworkQuery, NetworkListResponse, @@ -19,6 +21,8 @@ NetworkIDSerializer, ) from api.utils.common import with_common_response +from api.lib.configtxgen import ConfigTX, ConfigTxGen +from api.models import Network, Node, Organization LOG = logging.getLogger(__name__) @@ -32,10 +36,38 @@ class NetworkViewSet(viewsets.ViewSet): ) def list(self, request): """ - List Networks + List network - Filter networks with query parameters. + :param request: query parameter + :return: network list + :rtype: list """ + serializer = NetworkQuery(data=request.GET) + if serializer.is_valid(raise_exception=True): + page = serializer.validated_data.get("page", 1) + per_page = serializer.validated_data.get("per_page", 10) + name = serializer.validated_data.get("name") + parameters = {} + if name: + parameters.update({"name__icontains": name}) + networks = Network.objects.filter(**parameters) + p = Paginator(networks, per_page) + networks = p.page(page) + networks = [ + { + "id": str(network.id), + "name": network.name, + "created_at": network.created_at, + } + for network in networks + ] + response = NetworkListResponse( + data={"total": p.count, "data": networks} + ) + if response.is_valid(raise_exception=True): + return Response( + response.validated_data, status=status.HTTP_200_OK + ) return Response(data=[], status=status.HTTP_200_OK) @swagger_auto_schema( @@ -46,12 +78,48 @@ def list(self, request): ) def create(self, request): """ - New Network + Create Network - Create new network through internal nodes, - or import exists network outside + :param request: create parameter + :return: organization ID + :rtype: uuid """ - pass + serializer = NetworkCreateBody(data=request.data) + if serializer.is_valid(raise_exception=True): + name = serializer.validated_data.get("name") + consensus = serializer.validated_data.get("consensus") + organizations = serializer.validated_data.get("organizations") + + try: + Network.objects.get(name=name) + except ObjectDoesNotExist: + pass + orderers = [] + peers = [] + i = 0 + for organization in organizations: + org = Organization.objects.get(pk=organization) + orderers.append({"name": org.name, "hosts": []}) + peers.append({"name": org.name, "hosts": []}) + nodes = Node.objects.filter(org=org) + for node in nodes: + if node.type == "peer": + peers[i]["hosts"].append({"name": node.name, "port": node.urls.split(":")[2]}) + elif node.type == "orderer": + orderers[i]["hosts"].append({"name": node.name, "port": node.urls.split(":")[2]}) + i = i + 1 + + ConfigTX(name).create(consensus=consensus, orderers=orderers, peers=peers) + ConfigTxGen(name).genesis() + + network = Network(name=name, consensus=consensus, organizations=organizations) + network.save() + + response = NetworkIDSerializer(data=network.__dict__) + if response.is_valid(raise_exception=True): + return Response( + response.validated_data, status=status.HTTP_201_CREATED + ) @swagger_auto_schema(responses=with_common_response()) def retrieve(self, request, pk=None): @@ -62,6 +130,20 @@ def retrieve(self, request, pk=None): """ pass + @swagger_auto_schema( + responses=with_common_response( + {status.HTTP_204_NO_CONTENT: "No Content"} + ) + ) + def destroy(self, request, pk=None): + try: + network = Network.objects.get(pk=pk) + network.delete() + except ObjectDoesNotExist: + raise BaseException + + return Response(status=status.HTTP_204_NO_CONTENT) + @swagger_auto_schema( methods=["get"], responses=with_common_response( diff --git a/src/api-engine/api/routes/node/serializers.py b/src/api-engine/api/routes/node/serializers.py index 3dd8609b2..11588f660 100644 --- a/src/api-engine/api/routes/node/serializers.py +++ b/src/api-engine/api/routes/node/serializers.py @@ -42,8 +42,6 @@ class Meta: "per_page", "type", "name", - "network_type", - "network_version", "agent_id", ) extra_kwargs = {"type": {"required": False}} @@ -238,69 +236,56 @@ class Meta: class NodeCreateBody(serializers.ModelSerializer): - agent_type = serializers.ChoiceField( - help_text="Agent type", - choices=HostType.to_choices(True), - required=False, - ) - ca = FabricCASerializer( - help_text="CA configuration for node", required=False - ) - peer = FabricPeerSerializer( - help_text="Peer configuration for node", required=False - ) + organization = serializers.UUIDField(help_text="ID of organization") class Meta: model = Node fields = ( - "network_type", - "network_version", + "name", + "urls", "type", - "agent_type", - "agent", - "ca", - "peer", + "organization", ) extra_kwargs = { - "network_type": {"required": True}, - "network_version": {"required": True}, + "name": {"required": True}, + "urls": {"required": True}, "type": {"required": True}, } def validate(self, attrs): - network_type = attrs.get("network_type") - node_type = attrs.get("type") - network_version = attrs.get("network_version") - agent_type = attrs.get("agent_type") - agent = attrs.get("agent") - ca = attrs.get("ca") - peer = attrs.get("peer") - if network_type == NetworkType.Fabric.value: - if network_version not in FabricVersions.values(): - raise serializers.ValidationError("Not valid fabric version") - if node_type not in FabricNodeType.names(): - raise serializers.ValidationError( - "Not valid node type for %s" % network_type - ) - if node_type == FabricNodeType.Ca.name.lower() and ca is None: - raise serializers.ValidationError( - "Please input ca configuration for ca node" - ) - elif ( - node_type == FabricNodeType.Peer.name.lower() and peer is None - ): - raise serializers.ValidationError( - "Please input peer configuration for peer node" - ) - - if agent_type is None and agent is None: - raise serializers.ValidationError("Please set agent_type or agent") - - if agent_type and agent: - if agent_type != agent.type: - raise serializers.ValidationError( - "agent type not equal to agent" - ) + # network_type = attrs.get("network_type") + # node_type = attrs.get("type") + # network_version = attrs.get("network_version") + # agent_type = attrs.get("agent_type") + # agent = attrs.get("agent") + # ca = attrs.get("ca") + # peer = attrs.get("peer") + # if network_type == NetworkType.Fabric.value: + # if network_version not in FabricVersions.values(): + # raise serializers.ValidationError("Not valid fabric version") + # if node_type not in FabricNodeType.names(): + # raise serializers.ValidationError( + # "Not valid node type for %s" % network_type + # ) + # if node_type == FabricNodeType.Ca.name.lower() and ca is None: + # raise serializers.ValidationError( + # "Please input ca configuration for ca node" + # ) + # elif ( + # node_type == FabricNodeType.Peer.name.lower() and peer is None + # ): + # raise serializers.ValidationError( + # "Please input peer configuration for peer node" + # ) + # + # if agent_type is None and agent is None: + # raise serializers.ValidationError("Please set agent_type or agent") + # + # if agent_type and agent: + # if agent_type != agent.type: + # raise serializers.ValidationError( + # "agent type not equal to agent" + # ) return attrs diff --git a/src/api-engine/api/routes/node/views.py b/src/api-engine/api/routes/node/views.py index da6885737..775459e03 100644 --- a/src/api-engine/api/routes/node/views.py +++ b/src/api-engine/api/routes/node/views.py @@ -3,6 +3,7 @@ # import json import logging +import base64 from django.core.exceptions import ObjectDoesNotExist from django.core.exceptions import PermissionDenied @@ -23,6 +24,7 @@ from api.models import ( Agent, Node, + Organization, Port, FabricCA, FabricNodeType, @@ -50,21 +52,25 @@ from api.tasks import operate_node from api.utils.common import with_common_response from api.auth import CustomAuthenticate +from api.lib.pki import CryptoGen, CryptoConfig +from api.utils import zip_dir, zip_file +from api.config import CELLO_HOME + LOG = logging.getLogger(__name__) class NodeViewSet(viewsets.ViewSet): - authentication_classes = (CustomAuthenticate, JSONWebTokenAuthentication) + #authentication_classes = (CustomAuthenticate, JSONWebTokenAuthentication) # Only operator can update node info - def get_permissions(self): - if self.action in ["update"]: - permission_classes = (IsAuthenticated, IsOperatorAuthenticated) - else: - permission_classes = (IsAuthenticated,) - - return [permission() for permission in permission_classes] + # def get_permissions(self): + # if self.action in ["update"]: + # permission_classes = (IsAuthenticated, IsOperatorAuthenticated) + # else: + # permission_classes = (IsAuthenticated,) + # + # return [permission() for permission in permission_classes] @staticmethod def _validate_organization(request): @@ -79,9 +85,11 @@ def _validate_organization(request): ) def list(self, request, *args, **kwargs): """ - List Nodes + List node - Filter nodes with query parameters. + :param request: query parameter + :return: node list + :rtype: list """ serializer = NodeQuery(data=request.GET) if serializer.is_valid(raise_exception=True): @@ -89,28 +97,22 @@ def list(self, request, *args, **kwargs): per_page = serializer.validated_data.get("per_page") node_type = serializer.validated_data.get("type") name = serializer.validated_data.get("name") - network_type = serializer.validated_data.get("network_type") - network_version = serializer.validated_data.get("network_version") agent_id = serializer.validated_data.get("agent_id") - if agent_id is not None and not request.user.is_operator: - raise PermissionDenied + # if agent_id is not None and not request.user.is_operator: + # raise PermissionDenied query_filter = {} if node_type: query_filter.update({"type": node_type}) if name: query_filter.update({"name__icontains": name}) - if network_type: - query_filter.update({"network_type": network_type}) - if network_version: - query_filter.update({"network_version": network_version}) - if request.user.is_administrator: - query_filter.update( - {"organization": request.user.organization} - ) - elif request.user.is_common_user: - query_filter.update({"user": request.user}) + # if request.user.is_administrator: + # query_filter.update( + # {"organization": request.user.organization} + # ) + # elif request.user.is_common_user: + # query_filter.update({"user": request.user}) if agent_id: query_filter.update({"agent__id": agent_id}) nodes = Node.objects.filter(**query_filter) @@ -254,81 +256,76 @@ def create(self, request): """ Create Node - Create node + :param request: create parameter + :return: node ID + :rtype: uuid """ serializer = NodeCreateBody(data=request.data) if serializer.is_valid(raise_exception=True): - self._validate_organization(request) - agent_type = serializer.validated_data.get("agent_type") - network_type = serializer.validated_data.get("network_type") - network_version = serializer.validated_data.get("network_version") - agent = serializer.validated_data.get("agent") - node_type = serializer.validated_data.get("type") - ca = serializer.validated_data.get("ca") - peer = serializer.validated_data.get("peer") - if agent is None: - available_agents = ( - Agent.objects.annotate(network_num=Count("node__network")) - .annotate(node_num=Count("node")) - .filter( - schedulable=True, - type=agent_type, - network_num__lt=F("capacity"), - node_num__lt=F("node_capacity"), - organization=request.user.organization, - ) - .order_by("node_num") - ) - if len(available_agents) > 0: - agent = available_agents[0] - else: - raise NoResource + #self._validate_organization(request) + name = serializer.validated_data.get("name") + type = serializer.validated_data.get("type") + urls = serializer.validated_data.get("urls") + organization = serializer.validated_data.get("organization") + + org = Organization.objects.get(id=organization) + if org.organization.all(): + pass else: - if not request.user.is_operator: - raise PermissionDenied - node_count = Node.objects.filter(agent=agent).count() - if node_count >= agent.node_capacity or not agent.schedulable: - raise NoResource - - fabric_ca = None - fabric_peer = None - peer_ca_list = [] - if node_type == FabricNodeType.Ca.name.lower(): - fabric_ca = self._save_fabric_ca(request, ca) - elif node_type == FabricNodeType.Peer.name.lower(): - fabric_peer, peer_ca_list = self._save_fabric_peer( - request, peer - ) + raise NoResource + nodes = { + "type": type, + "Specs": [name] + } + CryptoConfig(org.name).update(nodes) + CryptoGen(org.name).extend() + + msp, tls = self._conversion_msp_tls(type, org.name, name) + node = Node( - network_type=network_type, - agent=agent, - network_version=network_version, - user=request.user, - organization=request.user.organization, - type=node_type, - ca=fabric_ca, - peer=fabric_peer, + name=name, + org=org, + urls=urls, + type=type, + msp=msp, + tls=tls ) node.save() - agent_config_file = request.build_absolute_uri( - agent.config_file.url - ) - node_detail_url = reverse("node-detail", args=[str(node.id)]) - node_detail_url = request.build_absolute_uri(node_detail_url) - node_file_upload_api = reverse("node-files", args=[str(node.id)]) - node_file_upload_api = request.build_absolute_uri( - node_file_upload_api - ) - operate_node.delay( - str(node.id), - AgentOperation.Create.value, - agent_config_file=agent_config_file, - node_detail_url=node_detail_url, - node_file_upload_api=node_file_upload_api, - peer_ca_list=json.dumps(peer_ca_list), - ) - response = NodeIDSerializer({"id": str(node.id)}) - return Response(response.data, status=status.HTTP_201_CREATED) + + response = NodeIDSerializer(data=node.__dict__) + if response.is_valid(raise_exception=True): + return Response( + response.validated_data, status=status.HTTP_201_CREATED + ) + + def _conversion_msp_tls(self, type, org, node): + """ + msp and tls from zip file to byte + + :param org: organization name + :param type: node type + :param node: node name + :return: msp, tls + :rtype: bytes + """ + try: + if type == "peer": + dir_node = "{}/{}/crypto-config/peerOrganizations/{}/peers/{}/" \ + .format(CELLO_HOME, org, org, node + "." + org) + else: + dir_node = "{}/{}/crypto-config/ordererOrganizations/{}/orderers/{}/" \ + .format(CELLO_HOME, org, org.split(".", 1)[1], node + "." + org.split(".", 1)[1]) + zip_dir("{}msp".format(dir_node), "{}msp.zip".format(dir_node)) + with open("{}msp.zip".format(dir_node), "rb") as f_msp: + msp = base64.b64encode(f_msp.read()) + + zip_dir("{}tls".format(dir_node), "{}tls.zip".format(dir_node)) + with open("{}tls.zip".format(dir_node), "rb") as f_tls: + tls = base64.b64encode(f_tls.read()) + except Exception as e: + raise e + + return msp, tls @swagger_auto_schema( methods=["post"], @@ -353,43 +350,19 @@ def destroy(self, request, pk=None): """ Delete Node - Delete node + :param request: destory parameter + :param pk: primary key + :return: none + :rtype: rest_framework.status """ try: - if request.user.is_superuser: - node = Node.objects.get(id=pk) - else: - node = Node.objects.get( - id=pk, organization=request.user.organization - ) + node = Node.objects.get(id=pk) + node.delete() + # todo delete node from agent except ObjectDoesNotExist: raise ResourceNotFound - else: - if node.status != NodeStatus.Deleting.name.lower(): - if node.status not in [ - NodeStatus.Error.name.lower(), - NodeStatus.Deleted.name.lower(), - ]: - node.status = NodeStatus.Deleting.name.lower() - node.save() - - agent_config_file = request.build_absolute_uri( - node.agent.config_file.url - ) - node_detail_url = reverse("node-detail", args=[pk]) - node_detail_url = request.build_absolute_uri( - node_detail_url - ) - operate_node.delay( - str(node.id), - AgentOperation.Delete.value, - agent_config_file=agent_config_file, - node_detail_url=node_detail_url, - ) - else: - node.delete() - return Response(status=status.HTTP_204_NO_CONTENT) + return Response(status=status.HTTP_204_NO_CONTENT) @swagger_auto_schema( operation_id="update node", diff --git a/src/api-engine/api/routes/organization/views.py b/src/api-engine/api/routes/organization/views.py index dbcbb72dc..672412373 100644 --- a/src/api-engine/api/routes/organization/views.py +++ b/src/api-engine/api/routes/organization/views.py @@ -2,6 +2,7 @@ # SPDX-License-Identifier: Apache-2.0 # import logging +import base64 from rest_framework import viewsets, status from rest_framework.decorators import action @@ -25,13 +26,18 @@ from api.routes.user.serializers import UserIDSerializer from api.models import UserProfile, Organization from api.routes.user.serializers import UserListSerializer, UserQuerySerializer +from api.lib.pki import CryptoGen, CryptoConfig +from api.utils import zip_dir, zip_file +from api.config import CELLO_HOME + LOG = logging.getLogger(__name__) class OrganizationViewSet(viewsets.ViewSet): - authentication_classes = (JSONWebTokenAuthentication,) - permission_classes = (IsAuthenticated, IsOperatorAuthenticated) + """Class represents orgnization related operations.""" + #authentication_classes = (JSONWebTokenAuthentication,) + #permission_classes = (IsAuthenticated, IsOperatorAuthenticated) @swagger_auto_schema( query_serializer=OrganizationQuery, @@ -43,7 +49,9 @@ def list(self, request): """ List Organizations - List organizations through query parameter + :param request: query parameter + :return: organization list + :rtype: list """ serializer = OrganizationQuery(data=request.GET) if serializer.is_valid(raise_exception=True): @@ -82,7 +90,9 @@ def create(self, request): """ Create Organization - Create Organization + :param request: create parameter + :return: organization ID + :rtype: uuid """ serializer = OrganizationCreateBody(data=request.data) if serializer.is_valid(raise_exception=True): @@ -93,7 +103,13 @@ def create(self, request): pass else: raise ResourceExists - organization = Organization(name=name) + + CryptoConfig(name).create() + CryptoGen(name).generate() + + msp, tls = self._conversion_msp_tls(name) + + organization = Organization(name=name, msp=msp, tls=tls) organization.save() response = OrganizationIDSerializer(data=organization.__dict__) @@ -102,6 +118,30 @@ def create(self, request): response.validated_data, status=status.HTTP_201_CREATED ) + def _conversion_msp_tls(self, name): + """ + msp and tls from zip file to byte + + :param name: organization name + :return: msp, tls + :rtype: bytes + """ + try: + dir_org = "{}/{}/crypto-config/peerOrganizations/{}/" \ + .format(CELLO_HOME, name, name) + + zip_dir("{}msp".format(dir_org), "{}msp.zip".format(dir_org)) + with open("{}msp.zip".format(dir_org), "rb") as f_msp: + msp = base64.b64encode(f_msp.read()) + + zip_dir("{}tlsca".format(dir_org), "{}tls.zip".format(dir_org)) + with open("{}tls.zip".format(dir_org), "rb") as f_tls: + tls = base64.b64encode(f_tls.read()) + except Exception as e: + raise e + + return msp, tls + @swagger_auto_schema( responses=with_common_response( {status.HTTP_204_NO_CONTENT: "No Content"} @@ -111,15 +151,21 @@ def destroy(self, request, pk=None): """ Delete Organization - Delete Organization + :param request: destory parameter + :param pk: primary key + :return: none + :rtype: rest_framework.status """ try: organization = Organization.objects.get(id=pk) - user_count = UserProfile.objects.filter( - organization=organization - ).count() - if user_count > 0: + if organization.network: raise ResourceInUse + + # user_count = UserProfile.objects.filter( + # organization=organization + # ).count() + # if user_count > 0: + # raise ResourceInUse organization.delete() except ObjectDoesNotExist: raise ResourceNotFound @@ -135,7 +181,10 @@ def retrieve(self, request, pk=None): """ Retrieve Organization - Retrieve Organization + :param request: retrieve parameter + :param pk: primary key + :return: organization info + :rtype: OrganizationResponse """ try: organization = Organization.objects.get(id=pk) diff --git a/src/api-engine/api/utils/__init__.py b/src/api-engine/api/utils/__init__.py index 8710d97a0..9b0d07bf1 100644 --- a/src/api-engine/api/utils/__init__.py +++ b/src/api-engine/api/utils/__init__.py @@ -8,6 +8,7 @@ from api.common.enums import ErrorCode from rest_framework import status from rest_framework.exceptions import ErrorDetail +from .common import zip_dir, zip_file LOG = logging.getLogger(__name__) diff --git a/src/api-engine/api/utils/common.py b/src/api-engine/api/utils/common.py index 35dbc5c38..e182f84f3 100644 --- a/src/api-engine/api/utils/common.py +++ b/src/api-engine/api/utils/common.py @@ -2,6 +2,7 @@ # SPDX-License-Identifier: Apache-2.0 # import hashlib +import os from drf_yasg import openapi from rest_framework import status @@ -10,6 +11,7 @@ from functools import reduce, partial from api.common.serializers import BadResponseSerializer import uuid +from zipfile import ZipFile def make_uuid(): @@ -99,3 +101,31 @@ def hash_file(file, block_size=65536): hash_func.update(buf) return hash_func.hexdigest() + + +def zip_dir(dirpath, outFullName): + """ + Compress the specified folder + :param dirpath: specified folder + :param outFullName: Save path+xxxx.zip + :return: null + """ + dir_dst = "/" + dirpath.rsplit("/", 1)[1] + zdir = ZipFile(outFullName, "w") + for path, dirnames, filenames in os.walk(dirpath): + fpath = dir_dst + path.replace(dirpath, '') + for filename in filenames: + zdir.write(os.path.join(path, filename), os.path.join(fpath, filename)) + zdir.close() + + +def zip_file(dirpath, outFullName): + """ + Compress the specified file + :param dirpath: specified folder of file + :param outFullName: Save path+filename.zip + :return: null + """ + zfile = ZipFile(outFullName, "w") + zfile.write(dirpath, dirpath.rsplit("/", 1)[1]) + zfile.close()