A 自定义发布支持发布时上传数据 #156

spug_api/apps/deploy/urls.py

@@ -7,5 +7,6 @@
 
 urlpatterns = [
     path('request/', RequestView.as_view()),
+    path('request/upload/', do_upload),
     path('request/<int:r_id>/', RequestDetailView.as_view()),
 ]

spug_api/apps/deploy/utils.py

@@ -136,7 +136,9 @@ def _ext2_deploy(req, helper, env):
     tmp_transfer_file = None
     for action in host_actions:
         if action.get('type') == 'transfer':
-            helper.send_info('local', f'{human_time()} 检测到数据传输动作，执行打包...   ')
+            if action.get('src_mode') == '1':
+                break
+            helper.send_info('local', f'{human_time()} 检测到来源为本地路径的数据传输动作，执行打包...   ')
             action['src'] = action['src'].rstrip('/ ')
             action['dst'] = action['dst'].rstrip('/ ')
             if not action['src'] or not action['dst']:
@@ -242,15 +244,23 @@ def _deploy_ext2_host(helper, h_id, actions, env):
     for index, action in enumerate(actions):
         helper.send_step(h_id, 2 + index, f'{human_time()} {action["title"]}...\r\n')
         if action.get('type') == 'transfer':
-            sp_dir, sd_dst = os.path.split(action['src'])
-            tar_gz_file = f'{env.SPUG_VERSION}.tar.gz'
-            try:
-                ssh.put_file(os.path.join(sp_dir, tar_gz_file), f'/tmp/{tar_gz_file}')
-            except Exception as e:
-                helper.send_error(host.id, f'exception: {e}')
-
-            command = f'cd /tmp && tar xf {tar_gz_file} && rm -f {tar_gz_file} '
-            command += f'&& rm -rf {action["dst"]} && mv /tmp/{sd_dst} {action["dst"]} && echo "transfer completed"'
+            if action.get('src_mode') == '1':
+                try:
+                    ssh.put_file(os.path.join(REPOS_DIR, env.SPUG_DEPLOY_ID, env.SPUG_VERSION), action['dst'])
+                except Exception as e:
+                    helper.send_error(host.id, f'exception: {e}')
+                helper.send_info(host.id, 'transfer completed\r\n')
+                continue
+            else:
+                sp_dir, sd_dst = os.path.split(action['src'])
+                tar_gz_file = f'{env.SPUG_VERSION}.tar.gz'
+                try:
+                    ssh.put_file(os.path.join(sp_dir, tar_gz_file), f'/tmp/{tar_gz_file}')
+                except Exception as e:
+                    helper.send_error(host.id, f'exception: {e}')
+
+                command = f'cd /tmp && tar xf {tar_gz_file} && rm -f {tar_gz_file} '
+                command += f'&& rm -rf {action["dst"]} && mv /tmp/{sd_dst} {action["dst"]} && echo "transfer completed"'
         else:
             command = f'cd /tmp && {action["data"]}'
         helper.remote(host.id, ssh, command, env)

spug_api/apps/deploy/views.py

@@ -4,17 +4,20 @@
 from django.views.generic import View
 from django.db.models import F
 from django.conf import settings
+from django.http.response import HttpResponseBadRequest
 from django_redis import get_redis_connection
 from libs import json_response, JsonParser, Argument, human_datetime, human_time
 from apps.deploy.models import DeployRequest
-from apps.app.models import Deploy
+from apps.app.models import Deploy, DeployExtend2
 from apps.deploy.utils import deploy_dispatch, Helper
 from apps.host.models import Host
 from collections import defaultdict
 from threading import Thread
 from datetime import datetime
+import subprocess
 import json
 import uuid
+import os
 
 
 class RequestView(View):
@@ -63,6 +66,11 @@ def post(self, request):
                 return json_response(error='请选择要发布的Tag')
             if form.extra[0] == 'branch' and not form.extra[2]:
                 return json_response(error='请选择要发布的分支及Commit ID')
+            if deploy.extend == '2':
+                if DeployExtend2.objects.filter(host_actions__contains='"src_mode": "1"').exists():
+                    if len(form.extra) < 2:
+                        return json_response(error='该应用的发布配置中使用了数据传输动作且设置为发布时上传，请上传要传输的数据')
+                    form.version = form.extra[1].get('path')
             form.status = '0' if deploy.is_audit else '1'
             form.extra = json.dumps(form.extra)
             form.host_ids = json.dumps(form.host_ids)
@@ -215,3 +223,22 @@ def patch(self, request, r_id):
             req.save()
             Thread(target=Helper.send_deploy_notify, args=(req, 'approve_rst')).start()
         return json_response(error=error)
+
+
+def do_upload(request):
+    repos_dir = settings.REPOS_DIR
+    file = request.FILES['file']
+    deploy_id = request.POST.get('deploy_id')
+    if file and deploy_id:
+        dir_name = os.path.join(repos_dir, deploy_id)
+        file_name = datetime.now().strftime("%Y%m%d%H%M%S")
+        command = f'mkdir -p {dir_name} && cd {dir_name} && ls | sort  -rn | tail -n +11 | xargs rm -rf'
+        code, outputs = subprocess.getstatusoutput(command)
+        if code != 0:
+            return json_response(error=outputs)
+        with open(os.path.join(dir_name, file_name), 'wb') as f:
+            for chunk in file.chunks():
+                f.write(chunk)
+        return json_response(file_name)
+    else:
+        return HttpResponseBadRequest()

spug_web/src/pages/deploy/app/Ext2Setup3.js

@@ -33,7 +33,7 @@ class Ext2Setup3 extends React.Component {
     const info = store.deploy;
     info['app_id'] = store.app_id;
     info['extend'] = '2';
-    info['host_actions'] = info['host_actions'].filter(x => (x.title && x.data) || (x.title && x.src && x.dst));
+    info['host_actions'] = info['host_actions'].filter(x => (x.title && x.data) || (x.title && (x.src || x.src_mode === '1') && x.dst));
     info['server_actions'] = info['server_actions'].filter(x => x.title && x.data);
     http.post('/api/app/deploy/', info)
       .then(res => {
@@ -101,31 +101,42 @@ class Ext2Setup3 extends React.Component {
                      placeholder="请输入"/>
             </Form.Item>
             {item['type'] === 'transfer' ? ([
-              <Form.Item key={0} label="过滤规则" help={this.helpMap[item['mode']]}>
+              <Form.Item key={0} required label="数据来源">
                 <Input
                   spellCheck={false}
-                  placeholder="请输入逗号分割的过滤规则"
-                  value={item['rule']}
-                  onChange={e => item['rule'] = e.target.value.replace('，', ',')}
-                  disabled={store.isReadOnly || item['mode'] === '0'}
+                  disabled={store.isReadOnly || item['src_mode'] === '1'}
+                  placeholder="请输入本地（部署spug的容器或主机）路径"
+                  value={item['src']}
+                  onChange={e => item['src'] = e.target.value}
                   addonBefore={(
-                    <Select disabled={store.isReadOnly} style={{width: 100}} value={item['mode']}
-                            onChange={v => item['mode'] = v}>
-                      <Select.Option value="0">关闭</Select.Option>
-                      <Select.Option value="1">包含</Select.Option>
-                      <Select.Option value="2">排除</Select.Option>
+                    <Select disabled={store.isReadOnly} style={{width: 120}} value={item['src_mode'] || '0'}
+                            onChange={v => item['src_mode'] = v}>
+                      <Select.Option value="0">本地路径</Select.Option>
+                      <Select.Option value="1">发布时上传</Select.Option>
                     </Select>
                   )}/>
               </Form.Item>,
-              <Form.Item key={1} required label="传输路径" extra={<a
+              item['src_mode'] === '0' ? (
+                <Form.Item key={1} label="过滤规则" help={this.helpMap[item['mode']]}>
+                  <Input
+                    spellCheck={false}
+                    placeholder="请输入逗号分割的过滤规则"
+                    value={item['rule']}
+                    onChange={e => item['rule'] = e.target.value.replace('，', ',')}
+                    disabled={store.isReadOnly || item['mode'] === '0'}
+                    addonBefore={(
+                      <Select disabled={store.isReadOnly} style={{width: 120}} value={item['mode']}
+                              onChange={v => item['mode'] = v}>
+                        <Select.Option value="0">关闭</Select.Option>
+                        <Select.Option value="1">包含</Select.Option>
+                        <Select.Option value="2">排除</Select.Option>
+                      </Select>
+                    )}/>
+                </Form.Item>
+              ) : null,
+              <Form.Item key={2} required label="目标路径" extra={<a
                 target="_blank" rel="noopener noreferrer"
                 href="https://spug.dev/docs/deploy-config#%E6%95%B0%E6%8D%AE%E4%BC%A0%E8%BE%93">使用前请务必阅读官方文档。</a>}>
-                <Input
-                  disabled={store.isReadOnly}
-                  spellCheck={false}
-                  value={item['src']}
-                  placeholder="请输入本地路径（部署spug的容器或主机）"
-                  onChange={e => item['src'] = e.target.value}/>
                 <Input
                   disabled={store.isReadOnly}
                   spellCheck={false}
@@ -162,7 +173,7 @@ class Ext2Setup3 extends React.Component {
               block
               type="dashed"
               disabled={store.isReadOnly || lds.findIndex(host_actions, x => x.type === 'transfer') !== -1}
-              onClick={() => host_actions.push({type: 'transfer', title: '数据传输', mode: '0'})}>
+              onClick={() => host_actions.push({type: 'transfer', title: '数据传输', mode: '0', src_mode: '0'})}>
               <Icon type="plus"/>添加数据传输动作（仅能添加一个）
             </Button>
           </Form.Item>

spug_web/src/pages/deploy/request/Ext2Form.js

@@ -5,7 +5,7 @@
  */
 import React from 'react';
 import { observer } from 'mobx-react';
-import { Modal, Form, Input, Tag, message } from 'antd';
+import { Modal, Form, Input, Tag, Upload, message, Button, Icon } from 'antd';
 import hostStore from 'pages/host/store';
 import http from 'libs/http';
 import store from './store';
@@ -15,9 +15,11 @@ import lds from 'lodash';
 class Ext2Form extends React.Component {
   constructor(props) {
     super(props);
+    this.token = localStorage.getItem('token');
     this.state = {
       loading: false,
-      type: null,
+      uploading: false,
+      fileList: [],
       host_ids: store.record['app_host_ids'].concat()
     }
   }
@@ -26,6 +28,11 @@ class Ext2Form extends React.Component {
     if (hostStore.records.length === 0) {
       hostStore.fetchRecords()
     }
+    const file = lds.get(store, 'record.extra.1');
+    if (file) {
+      file.uid = '0';
+      this.setState({fileList: [file]})
+    }
   }
 
   handleSubmit = () => {
@@ -37,6 +44,9 @@ class Ext2Form extends React.Component {
     formData['id'] = store.record.id;
     formData['deploy_id'] = store.record.deploy_id;
     formData['extra'] = [formData['extra']];
+    if (this.state.fileList.length > 0) {
+      formData['extra'].push(lds.pick(this.state.fileList[0], ['path', 'name']))
+    }
     formData['host_ids'] = this.state.host_ids;
     http.post('/api/deploy/request/', formData)
       .then(res => {
@@ -57,9 +67,28 @@ class Ext2Form extends React.Component {
     }
   };
 
+  handleUploadChange = (v) => {
+    if (v.fileList.length === 0) {
+      this.setState({fileList: []})
+    } else {
+      this.setState({fileList: [v.file]})
+    }
+  };
+
+  handleUpload = (file, fileList) => {
+    this.setState({uploading: true});
+    const formData = new FormData();
+    formData.append('file', file);
+    formData.append('deploy_id', store.record.deploy_id);
+    http.post('/api/deploy/request/upload/', formData)
+      .then(res => file.path = res)
+      .finally(() => this.setState({uploading: false}))
+    return false
+  };
+
   render() {
     const info = store.record;
-    const {host_ids} = this.state;
+    const {host_ids, fileList, uploading} = this.state;
     const {getFieldDecorator} = this.props.form;
     return (
       <Modal
@@ -81,10 +110,11 @@ class Ext2Form extends React.Component {
               <Input placeholder="请输入环境变量 SPUG_RELEASE 的值"/>
             )}
           </Form.Item>
-          <Form.Item label="备注信息">
-            {getFieldDecorator('desc', {initialValue: info['desc']})(
-              <Input placeholder="请输入备注信息"/>
-            )}
+          <Form.Item label="上传数据" help="通过数据传输动作来使用上传的文件。">
+            <Upload name="file" fileList={fileList} headers={{'X-Token': this.token}} beforeUpload={this.handleUpload}
+                    data={{deploy_id: info.deploy_id}} onChange={this.handleUploadChange}>
+              {fileList.length === 0 ? <Button loading={uploading}><Icon type="upload"/> 点击上传</Button> : null}
+            </Upload>
           </Form.Item>
           <Form.Item required label="发布目标主机" help="通过点击主机名称自由选择本次发布的主机。">
             {info['app_host_ids'].map(id => (