From fd42091f8768cd624c8bcff4b7dc763dcf7e5cd0 Mon Sep 17 00:00:00 2001 From: Arik Fraimovich Date: Tue, 26 Mar 2019 16:40:26 +0200 Subject: [PATCH 001/179] Add Lint step to CircleCI (#3642) --- .circleci/config.yml | 20 ++++- .codeclimate.yml | 24 ++---- client/.eslintrc.js | 65 ++++++++-------- client/app/.eslintrc.js | 7 ++ client/cypress/.eslintrc.js | 12 +++ .../integration/dashboard/dashboard_spec.js | 75 ++++++++++++------- client/cypress/integration/user/login_spec.js | 4 +- client/cypress/support/commands.js | 2 +- package-lock.json | 38 +++++----- package.json | 1 + 10 files changed, 147 insertions(+), 101 deletions(-) create mode 100644 client/app/.eslintrc.js create mode 100644 client/cypress/.eslintrc.js diff --git a/.circleci/config.yml b/.circleci/config.yml index 5788c5b52e..807580bdde 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -45,6 +45,16 @@ jobs: path: /tmp/test-results - store_artifacts: path: coverage.xml + frontend-lint: + docker: + - image: circleci/node:8 + steps: + - checkout + - run: mkdir -p /tmp/test-results/eslint + - run: npm install + - run: npm run lint:ci + - store_test_results: + path: /tmp/test-results frontend-unit-tests: docker: - image: circleci/node:8 @@ -54,6 +64,7 @@ jobs: - run: npm install - run: npm run bundle - run: npm test + - run: npm run lint frontend-e2e-tests: environment: COMPOSE_FILE: .circleci/docker-compose.cypress.yml @@ -105,8 +116,13 @@ workflows: - python-flake8-tests - legacy-python-flake8-tests - backend-unit-tests - - frontend-unit-tests - - frontend-e2e-tests + - frontend-lint + - frontend-unit-tests: + requires: + - frontend-lint + - frontend-e2e-tests: + requires: + - frontend-lint - build-tarball: requires: - backend-unit-tests diff --git a/.codeclimate.yml b/.codeclimate.yml index c7def082ae..22f2e1f0cf 100644 --- a/.codeclimate.yml +++ b/.codeclimate.yml @@ -21,20 +21,12 @@ plugins: pep8: enabled: true eslint: - enabled: true - channel: "eslint-5" - config: - config: client/.eslintrc.js - checks: - import/no-unresolved: - enabled: false - no-multiple-empty-lines: # TODO: Enable - enabled: false + enabled: false exclude_patterns: -- "tests/**/*.py" -- "migrations/**/*.py" -- "setup/**/*" -- "bin/**/*" -- "**/node_modules/" -- "client/dist/" -- "**/*.pyc" + - "tests/**/*.py" + - "migrations/**/*.py" + - "setup/**/*" + - "bin/**/*" + - "**/node_modules/" + - "client/dist/" + - "**/*.pyc" diff --git a/client/.eslintrc.js b/client/.eslintrc.js index ff41b8c6d3..083d993905 100644 --- a/client/.eslintrc.js +++ b/client/.eslintrc.js @@ -1,23 +1,20 @@ module.exports = { root: true, - extends: ["airbnb", "plugin:jest/recommended"], - plugins: ["jest", "cypress", "chai-friendly"], + extends: ["airbnb"], settings: { "import/resolver": "webpack" }, parser: "babel-eslint", env: { - "jest/globals": true, - "cypress/globals": true, - "browser": true, - "node": true + browser: true, + node: true }, rules: { // allow debugger during development - 'no-debugger': process.env.NODE_ENV === 'production' ? 2 : 0, - 'no-param-reassign': 0, - 'no-mixed-operators': 0, - 'no-underscore-dangle': 0, + "no-debugger": process.env.NODE_ENV === "production" ? 2 : 0, + "no-param-reassign": 0, + "no-mixed-operators": 0, + "no-underscore-dangle": 0, "no-use-before-define": ["error", "nofunc"], "prefer-destructuring": "off", "prefer-template": "off", @@ -27,38 +24,42 @@ module.exports = { "no-lonely-if": "off", "consistent-return": "off", "no-control-regex": "off", - 'no-multiple-empty-lines': 'warn', + "no-multiple-empty-lines": "warn", "no-script-url": "off", // some tags should have href="javascript:void(0)" - 'operator-linebreak': 'off', - 'react/destructuring-assignment': 'off', + "operator-linebreak": "off", + "react/destructuring-assignment": "off", "react/jsx-filename-extension": "off", - 'react/jsx-one-expression-per-line': 'off', + "react/jsx-one-expression-per-line": "off", "react/jsx-uses-react": "error", "react/jsx-uses-vars": "error", - 'react/jsx-wrap-multilines': 'warn', - 'react/no-access-state-in-setstate': 'warn', + "react/jsx-wrap-multilines": "warn", + "react/no-access-state-in-setstate": "warn", "react/prefer-stateless-function": "warn", "react/forbid-prop-types": "warn", "react/prop-types": "warn", "jsx-a11y/anchor-is-valid": "off", "jsx-a11y/click-events-have-key-events": "off", - "jsx-a11y/label-has-associated-control": ["warn", { - "controlComponents": true - }], + "jsx-a11y/label-has-associated-control": [ + "warn", + { + controlComponents: true + } + ], "jsx-a11y/label-has-for": "off", "jsx-a11y/no-static-element-interactions": "off", - "max-len": ['error', 120, 2, { - ignoreUrls: true, - ignoreComments: false, - ignoreRegExpLiterals: true, - ignoreStrings: true, - ignoreTemplateLiterals: true, - }], - "no-else-return": ["error", {"allowElseIf": true}], - "object-curly-newline": ["error", {"consistent": true}], - // needed for cypress tests - "func-names": "off", - "no-unused-expressions": 0, - "chai-friendly/no-unused-expressions": 2 + "max-len": [ + "error", + 120, + 2, + { + ignoreUrls: true, + ignoreComments: false, + ignoreRegExpLiterals: true, + ignoreStrings: true, + ignoreTemplateLiterals: true + } + ], + "no-else-return": ["error", { allowElseIf: true }], + "object-curly-newline": ["error", { consistent: true }] } }; diff --git a/client/app/.eslintrc.js b/client/app/.eslintrc.js new file mode 100644 index 0000000000..904b0635dc --- /dev/null +++ b/client/app/.eslintrc.js @@ -0,0 +1,7 @@ +module.exports = { + extends: ["plugin:jest/recommended"], + plugins: ["jest"], + env: { + "jest/globals": true, + }, +}; \ No newline at end of file diff --git a/client/cypress/.eslintrc.js b/client/cypress/.eslintrc.js new file mode 100644 index 0000000000..2049f84363 --- /dev/null +++ b/client/cypress/.eslintrc.js @@ -0,0 +1,12 @@ +module.exports = { + extends: ["plugin:cypress/recommended"], + plugins: ["cypress", "chai-friendly"], + env: { + "cypress/globals": true + }, + rules: { + "func-names": ["error", "never"], + "no-unused-expressions": 0, + "chai-friendly/no-unused-expressions": 2 + } +}; diff --git a/client/cypress/integration/dashboard/dashboard_spec.js b/client/cypress/integration/dashboard/dashboard_spec.js index cdb74061a6..3ae55a78a8 100644 --- a/client/cypress/integration/dashboard/dashboard_spec.js +++ b/client/cypress/integration/dashboard/dashboard_spec.js @@ -1,8 +1,7 @@ const DRAG_PLACEHOLDER_SELECTOR = '.grid-stack-placeholder'; function createNewDashboardByAPI(name) { - return cy.request('POST', 'api/dashboards', { name }) - .then(({ body }) => body); + return cy.request('POST', 'api/dashboards', { name }).then(({ body }) => body); } function editDashboard() { @@ -26,25 +25,28 @@ function addTextboxByAPI(text, dashId) { }, }; - return cy.request('POST', 'api/widgets', data) - .then(({ body }) => { - const id = Cypress._.get(body, 'id'); - assert.isDefined(id, 'Widget api call returns widget id'); - return `WidgetId${id}`; - }); + return cy.request('POST', 'api/widgets', data).then(({ body }) => { + const id = Cypress._.get(body, 'id'); + assert.isDefined(id, 'Widget api call returns widget id'); + return `WidgetId${id}`; + }); } function addQueryByAPI(data, shouldPublish = true) { - const merged = Object.assign({ - name: 'Test Query', - query: 'select 1', - data_source_id: 1, - options: { - parameters: [], + const merged = Object.assign( + { + name: 'Test Query', + query: 'select 1', + data_source_id: 1, + options: { + parameters: [], + }, + schedule: null, }, - schedule: null, - }, data); + data, + ); + // eslint-disable-next-line cypress/no-assigning-return-values const request = cy.request('POST', '/api/queries', merged); if (shouldPublish) { request.then(({ body }) => cy.request('POST', `/api/queries/${body.id}`, { is_draft: false })); @@ -109,7 +111,9 @@ describe('Dashboard', () => { it('creates new dashboard', () => { cy.visit('/dashboards'); cy.getByTestId('CreateButton').click(); - cy.get('li[role="menuitem"]').contains('Dashboard').click(); + cy.get('li[role="menuitem"]') + .contains('Dashboard') + .click(); cy.server(); cy.route('POST', 'api/dashboards').as('NewDashboard'); @@ -193,9 +197,13 @@ describe('Dashboard', () => { cy.visit(this.dashboardUrl); cy.getByTestId(elTestId) .within(() => { - cy.get('.widget-menu-regular').click({ force: true }).within(() => { - cy.get('li a').contains('Remove From Dashboard').click({ force: true }); - }); + cy.get('.widget-menu-regular') + .click({ force: true }) + .within(() => { + cy.get('li a') + .contains('Remove From Dashboard') + .click({ force: true }); + }); }) .should('not.exist'); }); @@ -235,18 +243,27 @@ describe('Dashboard', () => { it('edits textbox', function () { addTextboxByAPI('Hello World!', this.dashboardId).then((elTestId) => { cy.visit(this.dashboardUrl); - cy.getByTestId(elTestId).as('textboxEl') + cy.getByTestId(elTestId) + .as('textboxEl') .within(() => { - cy.get('.widget-menu-regular').click({ force: true }).within(() => { - cy.get('li a').contains('Edit').click({ force: true }); - }); + cy.get('.widget-menu-regular') + .click({ force: true }) + .within(() => { + cy.get('li a') + .contains('Edit') + .click({ force: true }); + }); }); const newContent = '[edited]'; - cy.get('edit-text-box').should('exist').within(() => { - cy.get('textarea').clear().type(newContent); - cy.contains('button', 'Save').click(); - }); + cy.get('edit-text-box') + .should('exist') + .within(() => { + cy.get('textarea') + .clear() + .type(newContent); + cy.contains('button', 'Save').click(); + }); cy.get('@textboxEl').should('contain', newContent); }); @@ -269,7 +286,7 @@ describe('Dashboard', () => { describe('Draggable', () => { describe('Grid snap', () => { - beforeEach(function () { + beforeEach(() => { editDashboard(); }); diff --git a/client/cypress/integration/user/login_spec.js b/client/cypress/integration/user/login_spec.js index 46cb45f67e..3e24f38269 100644 --- a/client/cypress/integration/user/login_spec.js +++ b/client/cypress/integration/user/login_spec.js @@ -6,7 +6,7 @@ describe('Login', () => { it('greets the user and take a screenshot', () => { cy.contains('h3', 'Login to Redash'); - cy.wait(1000); + cy.wait(1000); // eslint-disable-line cypress/no-unnecessary-waiting cy.percySnapshot('Login'); }); @@ -24,7 +24,7 @@ describe('Login', () => { cy.title().should('eq', 'Redash'); cy.contains('Example Admin'); - cy.wait(1000); + cy.wait(1000); // eslint-disable-line cypress/no-unnecessary-waiting cy.percySnapshot('Homepage'); }); }); diff --git a/client/cypress/support/commands.js b/client/cypress/support/commands.js index e8b39361b1..2ea4546b7e 100644 --- a/client/cypress/support/commands.js +++ b/client/cypress/support/commands.js @@ -1,4 +1,4 @@ -import '@percy/cypress'; // eslint-disable-line import/no-extraneous-dependencies +import '@percy/cypress'; // eslint-disable-line import/no-extraneous-dependencies, import/no-unresolved Cypress.Commands.add('login', (email = 'admin@redash.io', password = 'password') => cy.request({ url: '/login', diff --git a/package-lock.json b/package-lock.json index 437983243a..c2f8c28137 100644 --- a/package-lock.json +++ b/package-lock.json @@ -2479,7 +2479,7 @@ }, "bl": { "version": "1.2.2", - "resolved": "https://registry.npmjs.org/bl/-/bl-1.2.2.tgz", + "resolved": "http://registry.npmjs.org/bl/-/bl-1.2.2.tgz", "integrity": "sha512-e8tQYnZodmebYDWGH7KMRvtzKXaJHx3BbilrgZCfvyLUYdKpK1t5PSPmpkny/SgiTSCnjfLW7v5rlONXVFkQEA==", "requires": { "readable-stream": "^2.3.5", @@ -2493,7 +2493,7 @@ }, "readable-stream": { "version": "2.3.6", - "resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-2.3.6.tgz", + "resolved": "http://registry.npmjs.org/readable-stream/-/readable-stream-2.3.6.tgz", "integrity": "sha512-tQtKA9WIAhBF3+VLAseyMqZeBjW0AHJoxOtYqSUZNJxauErmLbVm2FW1y+J/YA9dUrAC39ITejlZWhVIwawkKw==", "requires": { "core-util-is": "~1.0.0", @@ -2705,7 +2705,7 @@ }, "readable-stream": { "version": "2.3.6", - "resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-2.3.6.tgz", + "resolved": "http://registry.npmjs.org/readable-stream/-/readable-stream-2.3.6.tgz", "integrity": "sha512-tQtKA9WIAhBF3+VLAseyMqZeBjW0AHJoxOtYqSUZNJxauErmLbVm2FW1y+J/YA9dUrAC39ITejlZWhVIwawkKw==", "requires": { "core-util-is": "~1.0.0", @@ -6251,7 +6251,7 @@ }, "finalhandler": { "version": "1.1.1", - "resolved": "https://registry.npmjs.org/finalhandler/-/finalhandler-1.1.1.tgz", + "resolved": "http://registry.npmjs.org/finalhandler/-/finalhandler-1.1.1.tgz", "integrity": "sha512-Y1GUDo39ez4aHAw7MysnUD5JzYX+WaIj8I57kO3aEPT1fFRL4sr7mjei97FgnwhAyyzRYmQZaTHb2+9uZ1dPtg==", "dev": true, "requires": { @@ -8738,7 +8738,7 @@ }, "readable-stream": { "version": "2.3.6", - "resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-2.3.6.tgz", + "resolved": "http://registry.npmjs.org/readable-stream/-/readable-stream-2.3.6.tgz", "integrity": "sha512-tQtKA9WIAhBF3+VLAseyMqZeBjW0AHJoxOtYqSUZNJxauErmLbVm2FW1y+J/YA9dUrAC39ITejlZWhVIwawkKw==", "requires": { "core-util-is": "~1.0.0", @@ -9080,7 +9080,7 @@ }, "readable-stream": { "version": "2.3.6", - "resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-2.3.6.tgz", + "resolved": "http://registry.npmjs.org/readable-stream/-/readable-stream-2.3.6.tgz", "integrity": "sha512-tQtKA9WIAhBF3+VLAseyMqZeBjW0AHJoxOtYqSUZNJxauErmLbVm2FW1y+J/YA9dUrAC39ITejlZWhVIwawkKw==", "dev": true, "requires": { @@ -9174,7 +9174,7 @@ }, "html-webpack-plugin": { "version": "3.2.0", - "resolved": "https://registry.npmjs.org/html-webpack-plugin/-/html-webpack-plugin-3.2.0.tgz", + "resolved": "http://registry.npmjs.org/html-webpack-plugin/-/html-webpack-plugin-3.2.0.tgz", "integrity": "sha1-sBq71yOsqqeze2r0SS69oD2d03s=", "dev": true, "requires": { @@ -9821,7 +9821,7 @@ }, "is-obj": { "version": "1.0.1", - "resolved": "https://registry.npmjs.org/is-obj/-/is-obj-1.0.1.tgz", + "resolved": "http://registry.npmjs.org/is-obj/-/is-obj-1.0.1.tgz", "integrity": "sha1-PkcprB9f3gJc19g6iW2rn09n2w8=" }, "is-observable": { @@ -11374,7 +11374,7 @@ }, "magic-string": { "version": "0.22.5", - "resolved": "https://registry.npmjs.org/magic-string/-/magic-string-0.22.5.tgz", + "resolved": "http://registry.npmjs.org/magic-string/-/magic-string-0.22.5.tgz", "integrity": "sha512-oreip9rJZkzvA8Qzk9HFs8fZGF/u7H/gtrE8EN6RjKJ9kh2HlC+yQ2QezifqTZfGyiuAV0dRv5a+y/8gBb1m9w==", "requires": { "vlq": "^0.2.2" @@ -11486,12 +11486,12 @@ }, "minimist": { "version": "0.0.8", - "resolved": "https://registry.npmjs.org/minimist/-/minimist-0.0.8.tgz", + "resolved": "http://registry.npmjs.org/minimist/-/minimist-0.0.8.tgz", "integrity": "sha1-hX/Kv8M5fSYluCKCYuhqp6ARsF0=" }, "readable-stream": { "version": "2.3.6", - "resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-2.3.6.tgz", + "resolved": "http://registry.npmjs.org/readable-stream/-/readable-stream-2.3.6.tgz", "integrity": "sha512-tQtKA9WIAhBF3+VLAseyMqZeBjW0AHJoxOtYqSUZNJxauErmLbVm2FW1y+J/YA9dUrAC39ITejlZWhVIwawkKw==", "requires": { "core-util-is": "~1.0.0", @@ -11681,7 +11681,7 @@ }, "readable-stream": { "version": "2.3.6", - "resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-2.3.6.tgz", + "resolved": "http://registry.npmjs.org/readable-stream/-/readable-stream-2.3.6.tgz", "integrity": "sha512-tQtKA9WIAhBF3+VLAseyMqZeBjW0AHJoxOtYqSUZNJxauErmLbVm2FW1y+J/YA9dUrAC39ITejlZWhVIwawkKw==", "dev": true, "requires": { @@ -11877,7 +11877,7 @@ }, "readable-stream": { "version": "2.3.6", - "resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-2.3.6.tgz", + "resolved": "http://registry.npmjs.org/readable-stream/-/readable-stream-2.3.6.tgz", "integrity": "sha512-tQtKA9WIAhBF3+VLAseyMqZeBjW0AHJoxOtYqSUZNJxauErmLbVm2FW1y+J/YA9dUrAC39ITejlZWhVIwawkKw==", "dev": true, "requires": { @@ -12370,7 +12370,7 @@ }, "readable-stream": { "version": "2.3.6", - "resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-2.3.6.tgz", + "resolved": "http://registry.npmjs.org/readable-stream/-/readable-stream-2.3.6.tgz", "integrity": "sha512-tQtKA9WIAhBF3+VLAseyMqZeBjW0AHJoxOtYqSUZNJxauErmLbVm2FW1y+J/YA9dUrAC39ITejlZWhVIwawkKw==", "dev": true, "requires": { @@ -12772,7 +12772,7 @@ "dependencies": { "minimist": { "version": "0.0.10", - "resolved": "https://registry.npmjs.org/minimist/-/minimist-0.0.10.tgz", + "resolved": "http://registry.npmjs.org/minimist/-/minimist-0.0.10.tgz", "integrity": "sha1-3j+YVD2/lggr5IrRoMfNqDYwHc8=", "dev": true }, @@ -15920,7 +15920,7 @@ "dependencies": { "minimist": { "version": "0.0.5", - "resolved": "https://registry.npmjs.org/minimist/-/minimist-0.0.5.tgz", + "resolved": "http://registry.npmjs.org/minimist/-/minimist-0.0.5.tgz", "integrity": "sha1-16oye87PUY+RBqxrjwA/o7zqhWY=" } } @@ -16408,7 +16408,7 @@ }, "split": { "version": "0.2.10", - "resolved": "https://registry.npmjs.org/split/-/split-0.2.10.tgz", + "resolved": "http://registry.npmjs.org/split/-/split-0.2.10.tgz", "integrity": "sha1-Zwl8YB1pfOE2j0GPBs0gHPBSGlc=", "requires": { "through": "2" @@ -16766,7 +16766,7 @@ "dependencies": { "readable-stream": { "version": "1.1.14", - "resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-1.1.14.tgz", + "resolved": "http://registry.npmjs.org/readable-stream/-/readable-stream-1.1.14.tgz", "integrity": "sha1-fPTFTvZI44EwhMY23SB54WbAgdk=", "requires": { "core-util-is": "~1.0.0", @@ -16900,7 +16900,7 @@ }, "strip-ansi": { "version": "3.0.1", - "resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-3.0.1.tgz", + "resolved": "http://registry.npmjs.org/strip-ansi/-/strip-ansi-3.0.1.tgz", "integrity": "sha1-ajhfuIU9lS1f8F0Oiq+UJ43GPc8=", "dev": true, "requires": { diff --git a/package.json b/package.json index 06332083ba..9f45fb0afb 100644 --- a/package.json +++ b/package.json @@ -13,6 +13,7 @@ "analyze:build": "npm run clean && NODE_ENV=production BUNDLE_ANALYZER=on webpack", "lint": "npm run lint:base -- --ext .js --ext .jsx ./client", "lint:base": "eslint --config ./client/.eslintrc.js --ignore-path ./client/.eslintignore", + "lint:ci": "npm run lint -- --format junit --output-file /tmp/test-results/eslint/results.xml", "test": "TZ=Africa/Khartoum jest", "test:watch": "jest --watch", "cypress:install": "npm install --no-save cypress@^3.1.5 @percy/cypress@^0.2.3 atob@2.1.2", From 8230098f50aa17745a74d62a2ca9e154b9eb4564 Mon Sep 17 00:00:00 2001 From: Ran Byron Date: Tue, 26 Mar 2019 19:23:00 +0200 Subject: [PATCH 002/179] Migrated Textbox edit dialog to React (#3632) --- ...AddTextboxDialog.jsx => TextboxDialog.jsx} | 31 +++++++---- ...dTextboxDialog.less => TextboxDialog.less} | 2 +- .../components/dashboards/edit-text-box.html | 18 ------- client/app/components/dashboards/widget.js | 51 +++---------------- client/app/pages/dashboards/dashboard.js | 4 +- .../integration/dashboard/dashboard_spec.js | 6 +-- 6 files changed, 34 insertions(+), 78 deletions(-) rename client/app/components/dashboards/{AddTextboxDialog.jsx => TextboxDialog.jsx} (81%) rename client/app/components/dashboards/{AddTextboxDialog.less => TextboxDialog.less} (93%) delete mode 100644 client/app/components/dashboards/edit-text-box.html diff --git a/client/app/components/dashboards/AddTextboxDialog.jsx b/client/app/components/dashboards/TextboxDialog.jsx similarity index 81% rename from client/app/components/dashboards/AddTextboxDialog.jsx rename to client/app/components/dashboards/TextboxDialog.jsx index b5d47ca999..ce1a4f10b1 100644 --- a/client/app/components/dashboards/AddTextboxDialog.jsx +++ b/client/app/components/dashboards/TextboxDialog.jsx @@ -9,20 +9,19 @@ import Divider from 'antd/lib/divider'; import { wrap as wrapDialog, DialogPropType } from '@/components/DialogWrapper'; import notification from '@/services/notification'; -import './AddTextboxDialog.less'; +import './TextboxDialog.less'; -class AddTextboxDialog extends React.Component { +class TextboxDialog extends React.Component { static propTypes = { dashboard: PropTypes.object.isRequired, // eslint-disable-line react/forbid-prop-types dialog: DialogPropType.isRequired, onConfirm: PropTypes.func.isRequired, + text: PropTypes.string, }; - state = { - saveInProgress: false, + static defaultProps = { text: '', - preview: '', - } + }; updatePreview = debounce(() => { const text = this.state.text; @@ -31,6 +30,16 @@ class AddTextboxDialog extends React.Component { }); }, 100); + constructor(props) { + super(props); + const { text } = props; + this.state = { + saveInProgress: false, + text, + preview: markdown.toHTML(text), + }; + } + onTextChanged = (event) => { this.setState({ text: event.target.value }); this.updatePreview(); @@ -53,20 +62,22 @@ class AddTextboxDialog extends React.Component { render() { const { dialog } = this.props; + const isNew = !this.props.text; return ( this.saveWidget()} okButtonProps={{ loading: this.state.saveInProgress, disabled: !this.state.text, }} - okText="Add to Dashboard" + okText={isNew ? 'Add to Dashboard' : 'Save'} width={500} + wrapProps={{ 'data-test': 'TextboxDialog' }} > -
+
- -

Edit TextBox

-
-
-
- -
-
- Preview: -

-
-
- -
- - -
diff --git a/client/app/components/dashboards/widget.js b/client/app/components/dashboards/widget.js index 76db96f1e4..c4d1439260 100644 --- a/client/app/components/dashboards/widget.js +++ b/client/app/components/dashboards/widget.js @@ -1,9 +1,8 @@ import { filter } from 'lodash'; import template from './widget.html'; -import editTextBoxTemplate from './edit-text-box.html'; +import TextboxDialog from '@/components/dashboards/TextboxDialog'; import widgetDialogTemplate from './widget-dialog.html'; import EditParameterMappingsDialog from '@/components/dashboards/EditParameterMappingsDialog'; -import notification from '@/services/notification'; import './widget.less'; import './widget-dialog.less'; @@ -19,51 +18,16 @@ const WidgetDialog = { }, }; - -const EditTextBoxComponent = { - template: editTextBoxTemplate, - bindings: { - resolve: '<', - close: '&', - dismiss: '&', - }, - controller() { - 'ngInject'; - - this.saveInProgress = false; - this.widget = this.resolve.widget; - this.saveWidget = () => { - this.saveInProgress = true; - if (this.widget.new_text !== this.widget.existing_text) { - this.widget.text = this.widget.new_text; - this.widget - .save() - .then(() => { - this.close(); - }) - .catch(() => { - notification.error('Widget can not be updated'); - }) - .finally(() => { - this.saveInProgress = false; - }); - } else { - this.close(); - } - }; - }, -}; - function DashboardWidgetCtrl($scope, $location, $uibModal, $window, $rootScope, $timeout, Events, currentUser) { this.canViewQuery = currentUser.hasPermission('view_query'); this.editTextBox = () => { - this.widget.existing_text = this.widget.text; - this.widget.new_text = this.widget.text; - $uibModal.open({ - component: 'editTextBox', - resolve: { - widget: this.widget, + TextboxDialog.showModal({ + dashboard: this.dashboard, + text: this.widget.text, + onConfirm: (text) => { + this.widget.text = text; + return this.widget.save(); }, }); }; @@ -143,7 +107,6 @@ function DashboardWidgetCtrl($scope, $location, $uibModal, $window, $rootScope, } export default function init(ngModule) { - ngModule.component('editTextBox', EditTextBoxComponent); ngModule.component('widgetDialog', WidgetDialog); ngModule.component('dashboardWidget', { template, diff --git a/client/app/pages/dashboards/dashboard.js b/client/app/pages/dashboards/dashboard.js index 1335ddc5a3..857d9e48d0 100644 --- a/client/app/pages/dashboards/dashboard.js +++ b/client/app/pages/dashboards/dashboard.js @@ -11,7 +11,7 @@ import { durationHumanize } from '@/filters'; import template from './dashboard.html'; import ShareDashboardDialog from './ShareDashboardDialog'; import AddWidgetDialog from '@/components/dashboards/AddWidgetDialog'; -import AddTextboxDialog from '@/components/dashboards/AddTextboxDialog'; +import TextboxDialog from '@/components/dashboards/TextboxDialog'; import notification from '@/services/notification'; import './dashboard.less'; @@ -327,7 +327,7 @@ function DashboardCtrl( }; this.showAddTextboxDialog = () => { - AddTextboxDialog.showModal({ + TextboxDialog.showModal({ dashboard: this.dashboard, onConfirm: this.addTextbox, }); diff --git a/client/cypress/integration/dashboard/dashboard_spec.js b/client/cypress/integration/dashboard/dashboard_spec.js index 3ae55a78a8..02a568d8f8 100644 --- a/client/cypress/integration/dashboard/dashboard_spec.js +++ b/client/cypress/integration/dashboard/dashboard_spec.js @@ -171,11 +171,11 @@ describe('Dashboard', () => { cy.visit(this.dashboardUrl); editDashboard(); cy.contains('a', 'Add Textbox').click(); - cy.get('.add-textbox').within(() => { + cy.getByTestId('TextboxDialog').within(() => { cy.get('textarea').type('Hello World!'); }); cy.contains('button', 'Add to Dashboard').click(); - cy.get('.add-textbox').should('not.exist'); + cy.getByTestId('TextboxDialog').should('not.exist'); cy.get('.textbox').should('exist'); }); @@ -256,7 +256,7 @@ describe('Dashboard', () => { }); const newContent = '[edited]'; - cy.get('edit-text-box') + cy.getByTestId('TextboxDialog') .should('exist') .within(() => { cy.get('textarea') From 73c8e3096dde1c1489d1f3c904cdcfb167bfe40a Mon Sep 17 00:00:00 2001 From: Levko Kravets Date: Wed, 27 Mar 2019 09:48:50 +0200 Subject: [PATCH 003/179] [Feature] Migrate Admin pages to React (#3568) --- client/app/components/admin/CeleryStatus.jsx | 267 ++++++------------ client/app/components/admin/Layout.jsx | 38 +++ client/app/components/admin/StatusBlock.jsx | 110 ++++++++ client/app/components/admin/layout.less | 17 ++ .../components/groups/DetailsPageSidebar.jsx | 1 + .../app/components/items-list/ItemsList.jsx | 7 +- .../items-list/classes/ItemsSource.js | 14 +- .../items-list/components/Sidebar.jsx | 24 +- client/app/config/antd-spinner.jsx | 4 + client/app/config/index.js | 1 + client/app/pages/admin/OutdatedQueries.jsx | 184 ++++++++++++ client/app/pages/admin/SystemStatus.jsx | 110 ++++++++ client/app/pages/admin/Tasks.jsx | 132 +++++++++ .../app/pages/admin/outdated-queries/index.js | 46 --- .../outdated-queries/outdated-queries.html | 50 ---- client/app/pages/admin/status/index.js | 44 --- client/app/pages/admin/status/status.html | 72 ----- client/app/pages/admin/system-status.less | 52 ++++ client/app/pages/admin/tasks/index.js | 24 -- client/app/pages/admin/tasks/tasks.html | 13 - client/app/pages/dashboards/DashboardList.jsx | 1 + client/app/pages/queries-list/QueriesList.jsx | 1 + client/app/pages/users/UsersList.jsx | 1 + 23 files changed, 760 insertions(+), 453 deletions(-) create mode 100644 client/app/components/admin/Layout.jsx create mode 100644 client/app/components/admin/StatusBlock.jsx create mode 100644 client/app/components/admin/layout.less create mode 100644 client/app/config/antd-spinner.jsx create mode 100644 client/app/pages/admin/OutdatedQueries.jsx create mode 100644 client/app/pages/admin/SystemStatus.jsx create mode 100644 client/app/pages/admin/Tasks.jsx delete mode 100644 client/app/pages/admin/outdated-queries/index.js delete mode 100644 client/app/pages/admin/outdated-queries/outdated-queries.html delete mode 100644 client/app/pages/admin/status/index.js delete mode 100644 client/app/pages/admin/status/status.html create mode 100644 client/app/pages/admin/system-status.less delete mode 100644 client/app/pages/admin/tasks/index.js delete mode 100644 client/app/pages/admin/tasks/tasks.html diff --git a/client/app/components/admin/CeleryStatus.jsx b/client/app/components/admin/CeleryStatus.jsx index b6c9fa8048..e21171ad9f 100644 --- a/client/app/components/admin/CeleryStatus.jsx +++ b/client/app/components/admin/CeleryStatus.jsx @@ -1,60 +1,16 @@ +import { map } from 'lodash'; import React from 'react'; import PropTypes from 'prop-types'; -import { $http } from '@/services/ng'; + import Table from 'antd/lib/table'; -import Col from 'antd/lib/col'; -import Row from 'antd/lib/row'; import Card from 'antd/lib/card'; import Spin from 'antd/lib/spin'; import Badge from 'antd/lib/badge'; -import Tabs from 'antd/lib/tabs'; -import Alert from 'antd/lib/alert'; -import moment from 'moment'; -import values from 'lodash/values'; import { Columns } from '@/components/items-list/components/ItemsTable'; -function parseTasks(tasks) { - const queues = {}; - const queries = []; - const otherTasks = []; - - const counters = { active: 0, reserved: 0, waiting: 0 }; - - tasks.forEach((task) => { - queues[task.queue] = queues[task.queue] || { name: task.queue, active: 0, reserved: 0, waiting: 0 }; - queues[task.queue][task.state] += 1; - - if (task.enqueue_time) { - task.enqueue_time = moment(task.enqueue_time * 1000.0); - } - if (task.start_time) { - task.start_time = moment(task.start_time * 1000.0); - } - - counters[task.state] += 1; - - if (task.task_name === 'redash.tasks.execute_query') { - queries.push(task); - } else { - otherTasks.push(task); - } - }); - - return { queues: values(queues), queries, otherTasks, counters }; -} - -function QueuesTable({ loading, queues }) { - const columns = ['Name', 'Active', 'Reserved', 'Waiting'].map(c => ({ title: c, dataIndex: c.toLowerCase() })); - - return ; -} - -QueuesTable.propTypes = { - loading: PropTypes.bool.isRequired, - queues: PropTypes.arrayOf(PropTypes.any).isRequired, -}; +// CounterCard -function CounterCard({ title, value, loading }) { +export function CounterCard({ title, value, loading }) { return ( @@ -75,145 +31,82 @@ CounterCard.defaultProps = { value: '', }; -export default class AdminCeleryStatus extends React.Component { - state = { - loading: true, - error: false, - counters: {}, - queries: [], - otherTasks: [], - queues: [], - }; - - constructor(props) { - super(props); - this.fetch(); - } - - fetch() { - // TODO: handle error - $http - .get('/api/admin/queries/tasks') - .then(({ data }) => { - const { queues, queries, otherTasks, counters } = parseTasks(data.tasks); - this.setState({ loading: false, queries, otherTasks, queues, counters }); - }) - .catch(() => { - this.setState({ loading: false, error: true }); - }); - } - - render() { - const commonColumns = [ - { - title: 'Worker Name', - dataIndex: 'worker', - }, - { - title: 'PID', - dataIndex: 'worker_pid', - }, - { - title: 'Queue', - dataIndex: 'queue', - }, - { - title: 'State', - dataIndex: 'state', - render: (value) => { - if (value === 'active') { - return ( - - Active - - ); - } - return ( - - {value} - - ); - }, - }, - Columns.timeAgo({ title: 'Start Time', dataIndex: 'start_time' }), - ]; - - const queryColumns = commonColumns.concat([ - Columns.timeAgo({ title: 'Enqueue Time', dataIndex: 'enqueue_time' }), - { - title: 'Query ID', - dataIndex: 'query_id', - }, - { - title: 'Org ID', - dataIndex: 'org_id', - }, - { - title: 'Data Source ID', - dataIndex: 'data_source_id', - }, - { - title: 'User ID', - dataIndex: 'user_id', - }, - { - title: 'Scheduled', - dataIndex: 'scheduled', - }, - ]); - - const otherTasksColumns = commonColumns.concat([ - { - title: 'Task Name', - dataIndex: 'task_name', - }, - ]); - - if (this.state.error) { - return ( -
- -
- ); +// Tables + +const commonColumns = [ + { title: 'Worker Name', dataIndex: 'worker' }, + { title: 'PID', dataIndex: 'worker_pid' }, + { title: 'Queue', dataIndex: 'queue' }, + Columns.custom((value) => { + if (value === 'active') { + return Active; } + return {value}; + }, { + title: 'State', + dataIndex: 'state', + }), + Columns.timeAgo({ title: 'Start Time', dataIndex: 'start_time' }), +]; + +const queryColumns = commonColumns.concat([ + Columns.timeAgo({ title: 'Enqueue Time', dataIndex: 'enqueue_time' }), + { title: 'Query ID', dataIndex: 'query_id' }, + { title: 'Org ID', dataIndex: 'org_id' }, + { title: 'Data Source ID', dataIndex: 'data_source_id' }, + { title: 'User ID', dataIndex: 'user_id' }, + { title: 'Scheduled', dataIndex: 'scheduled' }, +]); + +const otherTasksColumns = commonColumns.concat([ + { title: 'Task Name', dataIndex: 'task_name' }, +]); + +const queuesColumns = map( + ['Name', 'Active', 'Reserved', 'Waiting'], + c => ({ title: c, dataIndex: c.toLowerCase() }), +); + +const TablePropTypes = { + loading: PropTypes.bool.isRequired, + items: PropTypes.arrayOf(PropTypes.object).isRequired, +}; + +export function QueuesTable({ loading, items }) { + return ( +
+ ); +} + +QueuesTable.propTypes = TablePropTypes; - return ( -
- -
- - - - - - - - - - - - - - - -
- - -
- - - - - ); - } +export function QueriesTable({ loading, items }) { + return ( +
+ ); +} + +QueriesTable.propTypes = TablePropTypes; + +export function OtherTasksTable({ loading, items }) { + return ( +
+ ); } + +OtherTasksTable.propTypes = TablePropTypes; diff --git a/client/app/components/admin/Layout.jsx b/client/app/components/admin/Layout.jsx new file mode 100644 index 0000000000..505582da4e --- /dev/null +++ b/client/app/components/admin/Layout.jsx @@ -0,0 +1,38 @@ +import React from 'react'; +import PropTypes from 'prop-types'; +import Tabs from 'antd/lib/tabs'; +import { PageHeader } from '@/components/PageHeader'; + +import './layout.less'; + +export default function Layout({ activeTab, children }) { + return ( +
+ + +
+ + System Status}> + {(activeTab === 'system_status') ? children : null} + + Celery Status}> + {(activeTab === 'tasks') ? children : null} + + Outdated Queries}> + {(activeTab === 'outdated_queries') ? children : null} + + +
+
+ ); +} + +Layout.propTypes = { + activeTab: PropTypes.string, + children: PropTypes.node, +}; + +Layout.defaultProps = { + activeTab: 'system_status', + children: null, +}; diff --git a/client/app/components/admin/StatusBlock.jsx b/client/app/components/admin/StatusBlock.jsx new file mode 100644 index 0000000000..138f39cb07 --- /dev/null +++ b/client/app/components/admin/StatusBlock.jsx @@ -0,0 +1,110 @@ +/* eslint-disable react/prop-types */ + +import { toPairs } from 'lodash'; +import React from 'react'; + +import List from 'antd/lib/list'; +import Card from 'antd/lib/card'; +import { TimeAgo } from '@/components/TimeAgo'; + +import { toHuman, prettySize } from '@/filters'; + +export function General({ info }) { + info = toPairs(info); + return ( + + {(info.length === 0) && ( +
No data
+ )} + {(info.length > 0) && ( + ( + {value}}> + {toHuman(name)} + + )} + /> + )} + + ); +} + +export function DatabaseMetrics({ info }) { + return ( + + {(info.length === 0) && ( +
No data
+ )} + {(info.length > 0) && ( + ( + {prettySize(size)}}> + {name} + + )} + /> + )} + + ); +} + +export function Queues({ info }) { + info = toPairs(info); + return ( + + {(info.length === 0) && ( +
No data
+ )} + {(info.length > 0) && ( + ( + {queue.size}}> + {name} + + )} + /> + )} + + ); +} + +export function Manager({ info }) { + const items = info ? [( + }> + Last Refresh + + ), ( + }> + Started + + ), ( + {info.outdatedQueriesCount}}> + Outdated Queries Count + + )] : []; + + return ( + + {!info && ( +
No data
+ )} + {info && ( + item} + /> + )} + + ); +} diff --git a/client/app/components/admin/layout.less b/client/app/components/admin/layout.less new file mode 100644 index 0000000000..48cf38463f --- /dev/null +++ b/client/app/components/admin/layout.less @@ -0,0 +1,17 @@ +.admin-page-layout { + &-tabs.ant-tabs { + > .ant-tabs-bar { + margin: 0; + + .ant-tabs-tab { + padding: 0; + + a { + display: inline-block; + padding: 12px 16px; + color: inherit; + } + } + } + } +} diff --git a/client/app/components/groups/DetailsPageSidebar.jsx b/client/app/components/groups/DetailsPageSidebar.jsx index f0344e2eef..9ffdc8384d 100644 --- a/client/app/components/groups/DetailsPageSidebar.jsx +++ b/client/app/components/groups/DetailsPageSidebar.jsx @@ -21,6 +21,7 @@ export default function DetailsPageSidebar({ controller.updatePagination({ itemsPerPage })} diff --git a/client/app/components/items-list/ItemsList.jsx b/client/app/components/items-list/ItemsList.jsx index 34a0eaaad4..4a8cc79ff9 100644 --- a/client/app/components/items-list/ItemsList.jsx +++ b/client/app/components/items-list/ItemsList.jsx @@ -98,8 +98,11 @@ export function wrap(WrappedComponent, itemsSource, stateStorage) { } // eslint-disable-next-line class-methods-use-this - getState({ isLoaded, totalCount, pageItems, ...rest }) { - const params = { + getState({ isLoaded, totalCount, pageItems, params, ...rest }) { + params = { + // Custom params from items source + ...params, + // Add some properties of current route (`$resolve`, title, route params) // ANGULAR_REMOVE_ME Revisit when some React router will be used title: $route.current.title, diff --git a/client/app/components/items-list/classes/ItemsSource.js b/client/app/components/items-list/classes/ItemsSource.js index 93b592332a..e7ce3117a7 100644 --- a/client/app/components/items-list/classes/ItemsSource.js +++ b/client/app/components/items-list/classes/ItemsSource.js @@ -1,4 +1,4 @@ -import { isFunction, identity, map } from 'lodash'; +import { isFunction, identity, map, extend } from 'lodash'; import Paginator from './Paginator'; import Sorter from './Sorter'; import PromiseRejectionError from '@/lib/promise-rejection-error'; @@ -35,13 +35,20 @@ export class ItemsSource { searchTerm: this._searchTerm, selectedTags: this._selectedTags, }; - const context = this.getCallbackContext(); + const customParams = {}; + const context = { + ...this.getCallbackContext(), + setCustomParams: (params) => { + extend(customParams, params); + }, + }; return this._beforeUpdate().then(() => ( this._fetcher.fetch(changes, state, context) .then(({ results, count, allResults }) => { this._pageItems = results; this._allItems = allResults || null; this._paginator.setTotalCount(count); + this._params = { ...this._params, ...customParams }; return this._afterUpdate(); }) .catch((error) => { @@ -61,6 +68,8 @@ export class ItemsSource { this.setState(defaultState); this._pageItems = []; + + this._params = {}; } getState() { @@ -74,6 +83,7 @@ export class ItemsSource { totalCount: this._paginator.totalCount, pageItems: this._pageItems, allItems: this._allItems, + params: this._params, }; } diff --git a/client/app/components/items-list/components/Sidebar.jsx b/client/app/components/items-list/components/Sidebar.jsx index d8192ea8e6..89684a2274 100644 --- a/client/app/components/items-list/components/Sidebar.jsx +++ b/client/app/components/items-list/components/Sidebar.jsx @@ -139,20 +139,18 @@ Tags.propTypes = { PageSizeSelect */ -export function PageSizeSelect({ options, value, onChange }) { +export function PageSizeSelect({ options, value, onChange, ...props }) { return ( -
-
- -
+
+
); } diff --git a/client/app/config/antd-spinner.jsx b/client/app/config/antd-spinner.jsx new file mode 100644 index 0000000000..b26379b1b9 --- /dev/null +++ b/client/app/config/antd-spinner.jsx @@ -0,0 +1,4 @@ +import React from 'react'; +import Spin from 'antd/lib/spin'; + +Spin.setDefaultIndicator(); diff --git a/client/app/config/index.js b/client/app/config/index.js index d886abfe4e..8fa095a488 100644 --- a/client/app/config/index.js +++ b/client/app/config/index.js @@ -30,6 +30,7 @@ import registerDirectives from '@/directives'; import markdownFilter from '@/filters/markdown'; import dateTimeFilter from '@/filters/datetime'; import dashboardGridOptions from './dashboard-grid-options'; +import './antd-spinner'; const logger = debug('redash:config'); diff --git a/client/app/pages/admin/OutdatedQueries.jsx b/client/app/pages/admin/OutdatedQueries.jsx new file mode 100644 index 0000000000..319f0b1a28 --- /dev/null +++ b/client/app/pages/admin/OutdatedQueries.jsx @@ -0,0 +1,184 @@ +import { map } from 'lodash'; +import React from 'react'; +import { react2angular } from 'react2angular'; + +import Switch from 'antd/lib/switch'; +import * as Grid from 'antd/lib/grid'; +import { Paginator } from '@/components/Paginator'; +import { QueryTagsControl } from '@/components/tags-control/TagsControl'; +import { SchedulePhrase } from '@/components/queries/SchedulePhrase'; +import { TimeAgo } from '@/components/TimeAgo'; +import Layout from '@/components/admin/Layout'; + +import { wrap as itemsList, ControllerType } from '@/components/items-list/ItemsList'; +import { ItemsSource } from '@/components/items-list/classes/ItemsSource'; +import { StateStorage } from '@/components/items-list/classes/StateStorage'; + +import LoadingState from '@/components/items-list/components/LoadingState'; +import { PageSizeSelect } from '@/components/items-list/components/Sidebar'; +import ItemsTable, { Columns } from '@/components/items-list/components/ItemsTable'; + +import { $http } from '@/services/ng'; +import { Query } from '@/services/query'; +import recordEvent from '@/services/recordEvent'; +import { routesToAngularRoutes } from '@/lib/utils'; + +class OutdatedQueries extends React.Component { + static propTypes = { + controller: ControllerType.isRequired, + }; + + listColumns = [ + { + title: 'ID', + field: 'id', + width: '1%', + align: 'right', + sorter: true, + }, + Columns.custom.sortable((text, item) => ( + + {item.name} + + + ), { + title: 'Name', + field: 'name', + width: null, + }), + Columns.avatar({ field: 'user', className: 'p-l-0 p-r-0' }, name => `Created by ${name}`), + Columns.dateTime.sortable({ title: 'Created At', field: 'created_at' }), + Columns.duration.sortable({ title: 'Runtime', field: 'runtime' }), + Columns.dateTime.sortable({ title: 'Last Executed At', field: 'retrieved_at', orderByField: 'executed_at' }), + Columns.custom.sortable( + (text, item) => , + { title: 'Update Schedule', field: 'schedule' }, + ), + ]; + + state = { + autoUpdate: true, + }; + + _updateTimer = null; + + componentDidMount() { + recordEvent('view', 'page', 'admin/queries/outdated'); + this.update(true); + } + + componentWillUnmount() { + clearTimeout(this._updateTimer); + } + + update = (isInitialCall = false) => { + if (!isInitialCall && this.state.autoUpdate) { + this.props.controller.update(); + } + this._updateTimer = setTimeout(this.update, 60 * 1000); + }; + + render() { + const { controller } = this.props; + return ( + + + +
+ + this.setState({ autoUpdate })} + /> +
+ {controller.params.lastUpdatedAt && ( +
+ Last updated:{' '} + +
+ )} + + + {controller.isLoaded && !controller.isEmpty && ( + controller.updatePagination({ itemsPerPage })} + /> + )} + + + {!controller.isLoaded && } + {controller.isLoaded && controller.isEmpty && ( +
+ There are no outdated queries. +
+ )} + { + controller.isLoaded && !controller.isEmpty && ( +
+ + controller.updatePagination({ page })} + /> +
+ ) + } + + ); + } +} + +export default function init(ngModule) { + ngModule.component('pageOutdatedQueries', react2angular(itemsList( + OutdatedQueries, + new ItemsSource({ + doRequest(request, context) { + return $http.get('/api/admin/queries/outdated') + // eslint-disable-next-line camelcase + .then(({ data: { queries, updated_at } }) => { + context.setCustomParams({ lastUpdatedAt: parseFloat(updated_at) }); + return queries; + }); + }, + processResults(items) { + return map(items, item => new Query(item)); + }, + isPlainList: true, + }), + new StateStorage({ orderByField: 'created_at', orderByReverse: true }), + ))); + + return routesToAngularRoutes([ + { + path: '/admin/queries/outdated', + title: 'Outdated Queries', + key: 'outdated_queries', + }, + ], { + template: '', + controller($scope, $exceptionHandler) { + 'ngInject'; + + $scope.handleError = $exceptionHandler; + }, + }); +} + +init.init = true; diff --git a/client/app/pages/admin/SystemStatus.jsx b/client/app/pages/admin/SystemStatus.jsx new file mode 100644 index 0000000000..a86bab1a71 --- /dev/null +++ b/client/app/pages/admin/SystemStatus.jsx @@ -0,0 +1,110 @@ +import { omit } from 'lodash'; +import React from 'react'; +import PropTypes from 'prop-types'; +import { react2angular } from 'react2angular'; + +import Layout from '@/components/admin/Layout'; +import * as StatusBlock from '@/components/admin/StatusBlock'; + +import { $http } from '@/services/ng'; +import recordEvent from '@/services/recordEvent'; +import PromiseRejectionError from '@/lib/promise-rejection-error'; +import { routesToAngularRoutes } from '@/lib/utils'; + +import './system-status.less'; + +class SystemStatus extends React.Component { + static propTypes = { + onError: PropTypes.func, + }; + + static defaultProps = { + onError: () => {}, + }; + + state = { + queues: [], + manager: null, + databaseMetrics: {}, + status: {}, + }; + + _refreshTimer = null; + + componentDidMount() { + recordEvent('view', 'page', 'admin/status'); + this.refresh(); + } + + componentWillUnmount() { + clearTimeout(this._refreshTimer); + } + + refresh = () => { + $http.get('/status.json') + .then(({ data }) => { + this.setState({ + queues: data.manager.queues, + manager: { + startedAt: data.manager.started_at * 1000, + lastRefreshAt: data.manager.last_refresh_at * 1000, + outdatedQueriesCount: data.manager.outdated_queries_count, + }, + databaseMetrics: data.database_metrics.metrics || [], + status: omit(data, ['workers', 'manager', 'database_metrics']), + }); + }) + .catch((error) => { + // ANGULAR_REMOVE_ME This code is related to Angular's HTTP services + if (error.status && error.data) { + error = new PromiseRejectionError(error); + } + this.props.onError(error); + }); + this._refreshTimer = setTimeout(this.refresh, 60 * 1000); + }; + + render() { + return ( + +
+
+
+ +
+
+ +
+
+ +
+
+ +
+
+
+ + ); + } +} + +export default function init(ngModule) { + ngModule.component('pageSystemStatus', react2angular(SystemStatus)); + + return routesToAngularRoutes([ + { + path: '/admin/status', + title: 'System Status', + key: 'system_status', + }, + ], { + template: '', + controller($scope, $exceptionHandler) { + 'ngInject'; + + $scope.handleError = $exceptionHandler; + }, + }); +} + +init.init = true; diff --git a/client/app/pages/admin/Tasks.jsx b/client/app/pages/admin/Tasks.jsx new file mode 100644 index 0000000000..ea6a446a7e --- /dev/null +++ b/client/app/pages/admin/Tasks.jsx @@ -0,0 +1,132 @@ +import { values, each } from 'lodash'; +import moment from 'moment'; +import React from 'react'; +import { react2angular } from 'react2angular'; + +import Alert from 'antd/lib/alert'; +import Tabs from 'antd/lib/tabs'; +import * as Grid from 'antd/lib/grid'; +import Layout from '@/components/admin/Layout'; +import { CounterCard, QueuesTable, QueriesTable, OtherTasksTable } from '@/components/admin/CeleryStatus'; + +import { $http } from '@/services/ng'; +import recordEvent from '@/services/recordEvent'; +import { routesToAngularRoutes } from '@/lib/utils'; + +class Tasks extends React.Component { + state = { + isLoading: true, + error: null, + + queues: [], + queries: [], + otherTasks: [], + counters: { active: 0, reserved: 0, waiting: 0 }, + }; + + componentDidMount() { + recordEvent('view', 'page', 'admin/tasks'); + $http + .get('/api/admin/queries/tasks') + .then(({ data }) => this.processTasks(data.tasks)) + .catch(error => this.handleError(error)); + } + + componentWillUnmount() { + // Ignore data after component unmounted + this.processTasks = () => {}; + this.handleError = () => {}; + } + + processTasks = (tasks) => { + const queues = {}; + const queries = []; + const otherTasks = []; + + const counters = { active: 0, reserved: 0, waiting: 0 }; + + each(tasks, (task) => { + queues[task.queue] = queues[task.queue] || { name: task.queue, active: 0, reserved: 0, waiting: 0 }; + queues[task.queue][task.state] += 1; + + if (task.enqueue_time) { + task.enqueue_time = moment(task.enqueue_time * 1000.0); + } + if (task.start_time) { + task.start_time = moment(task.start_time * 1000.0); + } + + counters[task.state] += 1; + + if (task.task_name === 'redash.tasks.execute_query') { + queries.push(task); + } else { + otherTasks.push(task); + } + }); + + this.setState({ isLoading: false, queues: values(queues), queries, otherTasks, counters }); + }; + + handleError = (error) => { + this.setState({ isLoading: false, error }); + }; + + render() { + const { isLoading, error, queues, queries, otherTasks, counters } = this.state; + + return ( + +
+ {error && ( + + )} + + {!error && ( + + + + + + + + + + + + + + + + + + + + + + + + + + )} +
+ + ); + } +} + +export default function init(ngModule) { + ngModule.component('pageTasks', react2angular(Tasks)); + + return routesToAngularRoutes([ + { + path: '/admin/queries/tasks', + title: 'Celery Status', + key: 'tasks', + }, + ], { + template: '', + }); +} + +init.init = true; diff --git a/client/app/pages/admin/outdated-queries/index.js b/client/app/pages/admin/outdated-queries/index.js deleted file mode 100644 index 83fcb85320..0000000000 --- a/client/app/pages/admin/outdated-queries/index.js +++ /dev/null @@ -1,46 +0,0 @@ -import moment from 'moment'; - -import { Paginator } from '@/lib/pagination'; -import template from './outdated-queries.html'; - -function OutdatedQueriesCtrl($scope, $http, $timeout) { - $scope.autoUpdate = true; - - this.queries = new Paginator([], { itemsPerPage: 50 }); - - const refresh = () => { - if ($scope.autoUpdate) { - $scope.refresh_time = moment().add(1, 'minutes'); - $http.get('/api/admin/queries/outdated').success((data) => { - this.queries.updateRows(data.queries); - $scope.updatedAt = data.updated_at * 1000.0; - }); - } - - const timer = $timeout(refresh, 59 * 1000); - - $scope.$on('$destroy', () => { - if (timer) { - $timeout.cancel(timer); - } - }); - }; - - refresh(); -} - -export default function init(ngModule) { - ngModule.component('outdatedQueriesPage', { - template, - controller: OutdatedQueriesCtrl, - }); - - return { - '/admin/queries/outdated': { - template: '', - title: 'Outdated Queries', - }, - }; -} - -init.init = true; diff --git a/client/app/pages/admin/outdated-queries/outdated-queries.html b/client/app/pages/admin/outdated-queries/outdated-queries.html deleted file mode 100644 index cf0d3f3af4..0000000000 --- a/client/app/pages/admin/outdated-queries/outdated-queries.html +++ /dev/null @@ -1,50 +0,0 @@ -
- - -
- - -
-
- - - - - - - - - - - - - - - - - - - - - - -
IdNameCreated ByRuntimeLast Executed AtCreated AtUpdate Schedule
- {{row.data_source_id}} - - {{row.name}} - {{row.user.name}}{{row.runtime | durationHumanize}}{{row.retrieved_at | dateTime}}{{row.created_at | dateTime }}
- - -
-
- Last update: -
- () -
-
- - diff --git a/client/app/pages/admin/status/index.js b/client/app/pages/admin/status/index.js deleted file mode 100644 index a60b39c38b..0000000000 --- a/client/app/pages/admin/status/index.js +++ /dev/null @@ -1,44 +0,0 @@ -import template from './status.html'; - -// TODO: switch to $ctrl instead of $scope. -function AdminStatusCtrl($scope, $http, $timeout, currentUser, Events) { - Events.record('view', 'page', 'admin/status'); - - const refresh = () => { - $http.get('/status.json').success((data) => { - $scope.workers = data.workers; - delete data.workers; - $scope.manager = data.manager; - delete data.manager; - $scope.database_metrics = data.database_metrics; - delete data.database_metrics; - $scope.status = data; - }); - - const timer = $timeout(refresh, 59 * 1000); - - $scope.$on('$destroy', () => { - if (timer) { - $timeout.cancel(timer); - } - }); - }; - - refresh(); -} - -export default function init(ngModule) { - ngModule.component('statusPage', { - template, - controller: AdminStatusCtrl, - }); - - return { - '/admin/status': { - template: '', - title: 'System Status', - }, - }; -} - -init.init = true; diff --git a/client/app/pages/admin/status/status.html b/client/app/pages/admin/status/status.html deleted file mode 100644 index 701a3e8c59..0000000000 --- a/client/app/pages/admin/status/status.html +++ /dev/null @@ -1,72 +0,0 @@ -
- - -
- - -
-
    -
  • General
    • -
  • - {{ value }} - {{ name | toHuman }} -
    • -
- -
    -
  • Manager
    • -
  • - - Last Refresh -
    • -
  • - - Started -
    • -
  • - {{ manager.outdated_queries_count }} - Outdated Queries Count -
    • -
- -
    -
  • Queues
    • -
  • - {{ value.size }} - {{ name }} - - - -
    • -
-
-
-
    -
  • Redash Database
    • -
  • - {{ size[1] | prettySize }} - {{ size[0] }} -
    • -
-
-
-
diff --git a/client/app/pages/admin/system-status.less b/client/app/pages/admin/system-status.less new file mode 100644 index 0000000000..0171576269 --- /dev/null +++ b/client/app/pages/admin/system-status.less @@ -0,0 +1,52 @@ +.system-status-page { + @gutter: 15px; + + overflow: hidden; + padding: @gutter; + + .system-status-page-blocks { + display: flex; + align-items: stretch; + flex-wrap: wrap; + margin: -@gutter / 2; + + .system-status-page-block { + flex: 0 0 auto; + padding: @gutter / 2; + width: 100%; + + display: flex; + align-items: stretch; + + > div { + width: 100%; + } + + @media (min-width: 768px) { + & { + width: 50%; + } + } + + @media (min-width: 1600px) { + & { + width: 25%; + } + } + } + + .ant-list-item { + &:first-child { + padding-top: 0; + } + + &:last-child { + padding-bottom: 0; + } + + &-content { + margin: 0; + } + } + } +} diff --git a/client/app/pages/admin/tasks/index.js b/client/app/pages/admin/tasks/index.js deleted file mode 100644 index 4d30b20990..0000000000 --- a/client/app/pages/admin/tasks/index.js +++ /dev/null @@ -1,24 +0,0 @@ -import { react2angular } from 'react2angular'; -import CeleryStatus from '@/components/admin/CeleryStatus'; -import template from './tasks.html'; - -function TasksCtrl(Events) { - Events.record('view', 'page', 'admin/tasks'); -} - -export default function init(ngModule) { - ngModule.component('adminCeleryStatus', react2angular(CeleryStatus)); - ngModule.component('tasksPage', { - template, - controller: TasksCtrl, - }); - - return { - '/admin/queries/tasks': { - template: '', - title: 'Celery Status', - }, - }; -} - -init.init = true; diff --git a/client/app/pages/admin/tasks/tasks.html b/client/app/pages/admin/tasks/tasks.html deleted file mode 100644 index 860a98b2db..0000000000 --- a/client/app/pages/admin/tasks/tasks.html +++ /dev/null @@ -1,13 +0,0 @@ -
- - -
- - - -
-
diff --git a/client/app/pages/dashboards/DashboardList.jsx b/client/app/pages/dashboards/DashboardList.jsx index 12eb17fb95..bef99c2797 100644 --- a/client/app/pages/dashboards/DashboardList.jsx +++ b/client/app/pages/dashboards/DashboardList.jsx @@ -82,6 +82,7 @@ class DashboardList extends React.Component { controller.updatePagination({ itemsPerPage })} diff --git a/client/app/pages/queries-list/QueriesList.jsx b/client/app/pages/queries-list/QueriesList.jsx index 37dcb6e6b5..281a7f8845 100644 --- a/client/app/pages/queries-list/QueriesList.jsx +++ b/client/app/pages/queries-list/QueriesList.jsx @@ -98,6 +98,7 @@ class QueriesList extends React.Component { controller.updatePagination({ itemsPerPage })} diff --git a/client/app/pages/users/UsersList.jsx b/client/app/pages/users/UsersList.jsx index 925215fd40..111de07f96 100644 --- a/client/app/pages/users/UsersList.jsx +++ b/client/app/pages/users/UsersList.jsx @@ -194,6 +194,7 @@ class UsersList extends React.Component { /> controller.updatePagination({ itemsPerPage })} From 77c53130a456c4a2211df65d87aa30911e20e709 Mon Sep 17 00:00:00 2001 From: Jannis Leidel Date: Wed, 27 Mar 2019 16:14:32 +0100 Subject: [PATCH 004/179] Fix a few more inconsistencies when loading and dumping JSON. (#3626) * Fix a few more inconsistencies when loading and dumping JSON. Refs #2807. Original work in: #2817 These change have been added since c2429e92d2a5dd58cbe378b9cc06f3be969f747e. * Review fixes. --- redash/authentication/jwt_auth.py | 4 ++-- redash/monitor.py | 10 ++++------ redash/query_runner/db2.py | 8 +++----- redash/query_runner/kylin.py | 5 ++--- redash/query_runner/rockset.py | 6 ++---- redash/query_runner/uptycs.py | 7 +++---- 6 files changed, 16 insertions(+), 24 deletions(-) diff --git a/redash/authentication/jwt_auth.py b/redash/authentication/jwt_auth.py index 47c7870358..355591cb93 100644 --- a/redash/authentication/jwt_auth.py +++ b/redash/authentication/jwt_auth.py @@ -1,7 +1,7 @@ import logging -import json import jwt import requests +import simplejson logger = logging.getLogger('jwt_auth') @@ -21,7 +21,7 @@ def get_public_keys(url): if 'keys' in data: public_keys = [] for key_dict in data['keys']: - public_key = jwt.algorithms.RSAAlgorithm.from_jwk(json.dumps(key_dict)) + public_key = jwt.algorithms.RSAAlgorithm.from_jwk(simplejson.dumps(key_dict)) public_keys.append(public_key) get_public_keys.key_cache[url] = public_keys diff --git a/redash/monitor.py b/redash/monitor.py index c04235e0a8..877a1806b0 100644 --- a/redash/monitor.py +++ b/redash/monitor.py @@ -1,10 +1,8 @@ -import ast import itertools -import json -import base64 from sqlalchemy import union_all from redash import redis_connection, __version__, settings from redash.models import db, DataSource, Query, QueryResult, Dashboard, Widget +from redash.utils import json_loads from redash.worker import celery @@ -74,9 +72,9 @@ def get_status(): def get_waiting_in_queue(queue_name): jobs = [] for raw in redis_connection.lrange(queue_name, 0, -1): - job = json.loads(raw) + job = json_loads(raw) try: - args = json.loads(job['headers']['argsrepr']) + args = json_loads(job['headers']['argsrepr']) if args.get('query_id') == 'adhoc': args['query_id'] = None except ValueError: @@ -114,7 +112,7 @@ def parse_tasks(task_lists, state): if task['name'] == 'redash.tasks.execute_query': try: - args = json.loads(task['args']) + args = json_loads(task['args']) except ValueError: args = {} diff --git a/redash/query_runner/db2.py b/redash/query_runner/db2.py index 3253cee0b9..06a2de1f9c 100644 --- a/redash/query_runner/db2.py +++ b/redash/query_runner/db2.py @@ -1,9 +1,7 @@ -import os -import json import logging from redash.query_runner import * -from redash.utils import JSONEncoder +from redash.utils import json_dumps, json_loads logger = logging.getLogger(__name__) @@ -82,7 +80,7 @@ def _get_definitions(self, schema, query): if error is not None: raise Exception("Failed getting schema.") - results = json.loads(results) + results = json_loads(results) for row in results['rows']: if row['TABLE_SCHEMA'] != u'public': @@ -129,7 +127,7 @@ def run_query(self, query, user): data = {'columns': columns, 'rows': rows} error = None - json_data = json.dumps(data, cls=JSONEncoder) + json_data = json_dumps(data) else: error = 'Query completed but it returned no data.' json_data = None diff --git a/redash/query_runner/kylin.py b/redash/query_runner/kylin.py index a9f5d1fdb4..261fa3f5e0 100644 --- a/redash/query_runner/kylin.py +++ b/redash/query_runner/kylin.py @@ -1,12 +1,11 @@ import os -import json import logging import requests from requests.auth import HTTPBasicAuth from redash import settings from redash.query_runner import * -from redash.utils import JSONEncoder +from redash.utils import json_dumps logger = logging.getLogger(__name__) @@ -102,7 +101,7 @@ def run_query(self, query, user): columns = self.get_columns(data['columnMetas']) rows = self.get_rows(columns, data['results']) - return json.dumps({'columns': columns, 'rows': rows}), None + return json_dumps({'columns': columns, 'rows': rows}), None def get_schema(self, get_stats=False): url = self.configuration['url'] diff --git a/redash/query_runner/rockset.py b/redash/query_runner/rockset.py index 5d0d30d99d..eafb41c822 100644 --- a/redash/query_runner/rockset.py +++ b/redash/query_runner/rockset.py @@ -1,8 +1,6 @@ import requests -import os from redash.query_runner import * -from redash.utils import JSONEncoder -import json +from redash.utils import json_dumps def _get_type(value): @@ -96,7 +94,7 @@ def run_query(self, query, user): columns = [] for k in rows[0]: columns.append({'name': k, 'friendly_name': k, 'type': _get_type(rows[0][k])}) - data = json.dumps({'columns': columns, 'rows': rows}, cls=JSONEncoder) + data = json_dumps({'columns': columns, 'rows': rows}) return data, None diff --git a/redash/query_runner/uptycs.py b/redash/query_runner/uptycs.py index d8e8c59fb1..9e6e7ff989 100644 --- a/redash/query_runner/uptycs.py +++ b/redash/query_runner/uptycs.py @@ -1,7 +1,6 @@ from redash.query_runner import * -from redash.utils import json_dumps +from redash.utils import json_dumps, json_loads -import json import jwt import datetime import requests @@ -93,7 +92,7 @@ def api_call(self, sql): True)) if response.status_code == 200: - response_output = json.loads(response.content) + response_output = json_loads(response.content) else: error = 'status_code ' + str(response.status_code) + '\n' error = error + "failed to connect" @@ -124,7 +123,7 @@ def get_schema(self, get_stats=False): verify=self.configuration.get('verify_ssl', True)) redash_json = [] - schema = json.loads(response.content) + schema = json_loads(response.content) for each_def in schema['tables']: table_name = each_def['name'] columns = [] From 712fc63f9384344c32267e60c1db418b9356a003 Mon Sep 17 00:00:00 2001 From: Jannis Leidel Date: Wed, 27 Mar 2019 16:24:15 +0100 Subject: [PATCH 005/179] Use flask-talisman for handling backend response headers (#3404) * Normalize Flask initialization API use. * Use Flask-Talisman. * Enable HSTS when HTTPS is enforced. * More details about how CSP is formatted and write CSP directives as a string. * Use CSP frame-ancestors directive and not X-Frame-Options for embedable endpoints. * Add link to flask-talisman docs. * set remember_token cookie to be HTTP-Only and Secure * Reorganize secret key configuration to be forward thinking and backward compatible. --- redash/__init__.py | 13 ++--- redash/authentication/__init__.py | 1 - redash/authentication/account.py | 3 +- redash/extensions.py | 2 +- redash/handlers/__init__.py | 2 + redash/handlers/dashboards.py | 5 +- redash/handlers/embed.py | 4 +- redash/handlers/static.py | 5 +- redash/metrics/request.py | 2 +- redash/models/__init__.py | 2 +- redash/security.py | 43 +++++++++++++++ redash/settings/__init__.py | 84 +++++++++++++++++++++++++++-- requirements.txt | 2 +- setup/generate_key.sh | 2 + setup/setup.sh | 4 +- tests/handlers/test_data_sources.py | 2 +- tests/handlers/test_embed.py | 16 ++++++ tests/handlers/test_favorites.py | 3 +- tests/handlers/test_queries.py | 3 +- tests/handlers/test_settings.py | 5 +- 20 files changed, 168 insertions(+), 35 deletions(-) create mode 100644 redash/security.py diff --git a/redash/__init__.py b/redash/__init__.py index c1bf221b67..cc50844c89 100644 --- a/redash/__init__.py +++ b/redash/__init__.py @@ -5,7 +5,6 @@ import redis from flask import Flask, current_app -from flask_sslify import SSLify from werkzeug.contrib.fixers import ProxyFix from werkzeug.routing import BaseConverter from statsd import StatsClient @@ -98,11 +97,11 @@ def to_url(self, value): def create_app(): - from redash import authentication, extensions, handlers + from redash import authentication, extensions, handlers, security from redash.handlers.webpack import configure_webpack from redash.handlers import chrome_logger from redash.models import db, users - from redash.metrics.request import provision_app + from redash.metrics import request as request_metrics from redash.utils import sentry sentry.init() @@ -116,14 +115,12 @@ def create_app(): app.wsgi_app = ProxyFix(app.wsgi_app, settings.PROXIES_COUNT) app.url_map.converters['org_slug'] = SlugConverter - if settings.ENFORCE_HTTPS: - SSLify(app, skips=['ping']) - # configure our database app.config['SQLALCHEMY_DATABASE_URI'] = settings.SQLALCHEMY_DATABASE_URI app.config.update(settings.all_settings()) - provision_app(app) + security.init_app(app) + request_metrics.init_app(app) db.init_app(app) migrate.init_app(app, db) mail.init_app(app) @@ -131,7 +128,7 @@ def create_app(): limiter.init_app(app) handlers.init_app(app) configure_webpack(app) - extensions.init_extensions(app) + extensions.init_app(app) chrome_logger.init_app(app) users.init_app(app) diff --git a/redash/authentication/__init__.py b/redash/authentication/__init__.py index ce3ab982b6..eeb9a1d7c6 100644 --- a/redash/authentication/__init__.py +++ b/redash/authentication/__init__.py @@ -244,7 +244,6 @@ def init_app(app): login_manager.init_app(app) login_manager.anonymous_user = models.AnonymousUser - app.secret_key = settings.COOKIE_SECRET app.register_blueprint(google_oauth.blueprint) app.register_blueprint(saml_auth.blueprint) app.register_blueprint(remote_user_auth.blueprint) diff --git a/redash/authentication/account.py b/redash/authentication/account.py index 6a708d7763..3300796221 100644 --- a/redash/authentication/account.py +++ b/redash/authentication/account.py @@ -4,12 +4,11 @@ from redash import settings from redash.tasks import send_mail from redash.utils import base_url -from redash.models import User # noinspection PyUnresolvedReferences from itsdangerous import URLSafeTimedSerializer, SignatureExpired, BadSignature logger = logging.getLogger(__name__) -serializer = URLSafeTimedSerializer(settings.COOKIE_SECRET) +serializer = URLSafeTimedSerializer(settings.SECRET_KEY) def invite_token(user): diff --git a/redash/extensions.py b/redash/extensions.py index 53738a4514..dc3326bf2f 100644 --- a/redash/extensions.py +++ b/redash/extensions.py @@ -2,7 +2,7 @@ from pkg_resources import iter_entry_points, resource_isdir, resource_listdir -def init_extensions(app): +def init_app(app): """ Load the Redash extensions for the given Redash Flask app. """ diff --git a/redash/handlers/__init__.py b/redash/handlers/__init__.py index faf57c4a50..8de5cedaac 100644 --- a/redash/handlers/__init__.py +++ b/redash/handlers/__init__.py @@ -5,9 +5,11 @@ from redash.handlers.base import routes from redash.monitor import get_status from redash.permissions import require_super_admin +from redash.security import talisman @routes.route('/ping', methods=['GET']) +@talisman(force_https=False) def ping(): return 'PONG.' diff --git a/redash/handlers/dashboards.py b/redash/handlers/dashboards.py index 2df218a318..7fe147b5aa 100644 --- a/redash/handlers/dashboards.py +++ b/redash/handlers/dashboards.py @@ -6,10 +6,11 @@ from redash.handlers.base import (BaseResource, get_object_or_404, paginate, filter_by_tags, order_results as _order_results) -from redash.serializers import serialize_dashboard from redash.permissions import (can_modify, require_admin_or_owner, require_object_modify_permission, require_permission) +from redash.security import csp_allows_embeding +from redash.serializers import serialize_dashboard from sqlalchemy.orm.exc import StaleDataError @@ -235,6 +236,8 @@ def delete(self, dashboard_slug): class PublicDashboardResource(BaseResource): + decorators = BaseResource.decorators + [csp_allows_embeding] + def get(self, token): """ Retrieve a public dashboard. diff --git a/redash/handlers/embed.py b/redash/handlers/embed.py index dea0e5e0d0..62805f3a5d 100644 --- a/redash/handlers/embed.py +++ b/redash/handlers/embed.py @@ -9,10 +9,12 @@ from redash.handlers.base import (get_object_or_404, org_scoped_rule, record_event) from redash.handlers.static import render_index +from redash.security import csp_allows_embeding @routes.route(org_scoped_rule('/embed/query//visualization/'), methods=['GET']) @login_required +@csp_allows_embeding def embed(query_id, visualization_id, org_slug=None): record_event(current_org, current_user._get_current_object(), { 'action': 'view', @@ -22,12 +24,12 @@ def embed(query_id, visualization_id, org_slug=None): 'embed': True, 'referer': request.headers.get('Referer') }) - return render_index() @routes.route(org_scoped_rule('/public/dashboards/'), methods=['GET']) @login_required +@csp_allows_embeding def public_dashboard(token, org_slug=None): if current_user.is_api_user(): dashboard = current_user.object diff --git a/redash/handlers/static.py b/redash/handlers/static.py index 9c7711f2f4..1a386f4930 100644 --- a/redash/handlers/static.py +++ b/redash/handlers/static.py @@ -1,7 +1,4 @@ -import os - -from flask import current_app, render_template, safe_join, send_file -from werkzeug.exceptions import NotFound +from flask import render_template, safe_join, send_file from flask_login import login_required from redash import settings diff --git a/redash/metrics/request.py b/redash/metrics/request.py index 480625ac46..b90f81aaff 100644 --- a/redash/metrics/request.py +++ b/redash/metrics/request.py @@ -45,7 +45,7 @@ def calculate_metrics_on_exception(error): calculate_metrics(MockResponse(500, '?', -1)) -def provision_app(app): +def init_app(app): app.before_request(record_requets_start_time) app.after_request(calculate_metrics) app.teardown_request(calculate_metrics_on_exception) diff --git a/redash/models/__init__.py b/redash/models/__init__.py index ef9d3f01ef..ec642e2e73 100644 --- a/redash/models/__init__.py +++ b/redash/models/__init__.py @@ -74,7 +74,7 @@ class DataSource(BelongsToOrgMixin, db.Model): name = Column(db.String(255)) type = Column(db.String(255)) - options = Column('encrypted_options', ConfigurationContainer.as_mutable(EncryptedConfiguration(db.Text, settings.SECRET_KEY, FernetEngine))) + options = Column('encrypted_options', ConfigurationContainer.as_mutable(EncryptedConfiguration(db.Text, settings.DATASOURCE_SECRET_KEY, FernetEngine))) queue_name = Column(db.String(255), default="queries") scheduled_queue_name = Column(db.String(255), default="scheduled_queries") created_at = Column(db.DateTime(True), default=db.func.now()) diff --git a/redash/security.py b/redash/security.py new file mode 100644 index 0000000000..dc753008c2 --- /dev/null +++ b/redash/security.py @@ -0,0 +1,43 @@ +import functools +from flask_talisman import talisman + +from redash import settings + + +talisman = talisman.Talisman() + + +def csp_allows_embeding(fn): + + @functools.wraps(fn) + def decorated(*args, **kwargs): + return fn(*args, **kwargs) + + embedable_csp = talisman.content_security_policy + "frame-ancestors *;" + return talisman( + content_security_policy=embedable_csp, + frame_options=None, + )(decorated) + + +def init_app(app): + talisman.init_app( + app, + feature_policy=settings.FEATURE_POLICY, + force_https=settings.ENFORCE_HTTPS, + force_https_permanent=settings.ENFORCE_HTTPS_PERMANENT, + force_file_save=settings.ENFORCE_FILE_SAVE, + frame_options=settings.FRAME_OPTIONS, + frame_options_allow_from=settings.FRAME_OPTIONS_ALLOW_FROM, + strict_transport_security=settings.HSTS_ENABLED, + strict_transport_security_preload=settings.HSTS_PRELOAD, + strict_transport_security_max_age=settings.HSTS_MAX_AGE, + strict_transport_security_include_subdomains=settings.HSTS_INCLUDE_SUBDOMAINS, + content_security_policy=settings.CONTENT_SECURITY_POLICY, + content_security_policy_report_uri=settings.CONTENT_SECURITY_POLICY_REPORT_URI, + content_security_policy_report_only=settings.CONTENT_SECURITY_POLICY_REPORT_ONLY, + content_security_policy_nonce_in=settings.CONTENT_SECURITY_POLICY_NONCE_IN, + referrer_policy=settings.REFERRER_POLICY, + session_cookie_secure=settings.SESSION_COOKIE_SECURE, + session_cookie_http_only=settings.SESSION_COOKIE_HTTPONLY, + ) diff --git a/redash/settings/__init__.py b/redash/settings/__init__.py index 404b2baa91..21f7c86855 100644 --- a/redash/settings/__init__.py +++ b/redash/settings/__init__.py @@ -1,5 +1,6 @@ import os from funcy import distinct, remove +from flask_talisman import talisman from .helpers import fix_assets_path, array_from_string, parse_boolean, int_or_none, set_from_string from .organization import DATE_FORMAT @@ -15,7 +16,6 @@ def all_settings(): return settings - REDIS_URL = os.environ.get('REDASH_REDIS_URL', os.environ.get('REDIS_URL', "redis://localhost:6379/0")) PROXIES_COUNT = int(os.environ.get('REDASH_PROXIES_COUNT', "1")) @@ -50,9 +50,86 @@ def all_settings(): SCHEMAS_REFRESH_QUEUE = os.environ.get("REDASH_SCHEMAS_REFRESH_QUEUE", "celery") AUTH_TYPE = os.environ.get("REDASH_AUTH_TYPE", "api_key") -ENFORCE_HTTPS = parse_boolean(os.environ.get("REDASH_ENFORCE_HTTPS", "false")) INVITATION_TOKEN_MAX_AGE = int(os.environ.get("REDASH_INVITATION_TOKEN_MAX_AGE", 60 * 60 * 24 * 7)) +# The secret key to use in the Flask app for various cryptographic features +SECRET_KEY = os.environ.get("REDASH_COOKIE_SECRET", "c292a0a3aa32397cdb050e233733900f") +# The secret key to use when encrypting data source options +DATASOURCE_SECRET_KEY = os.environ.get('REDASH_SECRET_KEY', SECRET_KEY) + +# Whether and how to redirect non-HTTP requests to HTTPS. Disabled by default. +ENFORCE_HTTPS = parse_boolean(os.environ.get("REDASH_ENFORCE_HTTPS", "false")) +ENFORCE_HTTPS_PERMANENT = parse_boolean( + os.environ.get("REDASH_ENFORCE_HTTPS_PERMANENT", "false")) +# Whether file downloads are enforced or not. +ENFORCE_FILE_SAVE = parse_boolean( + os.environ.get("REDASH_ENFORCE_FILE_SAVE", "true")) + +# Whether to use secure cookies by default. +COOKIES_SECURE = parse_boolean( + os.environ.get("REDASH_COOKIES_SECURE", str(ENFORCE_HTTPS))) +# Whether the session cookie is set to secure. +SESSION_COOKIE_SECURE = parse_boolean( + os.environ.get("REDASH_SESSION_COOKIE_SECURE") or str(COOKIES_SECURE)) +# Whether the session cookie is set HttpOnly. +SESSION_COOKIE_HTTPONLY = parse_boolean( + os.environ.get("REDASH_SESSION_COOKIE_HTTPONLY", "true")) +# Whether the session cookie is set to secure. +REMEMBER_COOKIE_SECURE = parse_boolean( + os.environ.get("REDASH_REMEMBER_COOKIE_SECURE") or str(COOKIES_SECURE)) +# Whether the remember cookie is set HttpOnly. +REMEMBER_COOKIE_HTTPONLY = parse_boolean( + os.environ.get("REDASH_REMEMBER_COOKIE_HTTPONLY", "true")) + +# Doesn't set X-Frame-Options by default since it's highly dependent +# on the specific deployment. +# See https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers/X-Frame-Options +# for more information. +FRAME_OPTIONS = os.environ.get("REDASH_FRAME_OPTIONS", "deny") +FRAME_OPTIONS_ALLOW_FROM = os.environ.get( + "REDASH_FRAME_OPTIONS_ALLOW_FROM", "") + +# Whether and how to send Strict-Transport-Security response headers. +# See https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers/Strict-Transport-Security +# for more information. +HSTS_ENABLED = parse_boolean( + os.environ.get("REDASH_HSTS_ENABLED") or str(ENFORCE_HTTPS)) +HSTS_PRELOAD = parse_boolean(os.environ.get("REDASH_HSTS_PRELOAD", "false")) +HSTS_MAX_AGE = int( + os.environ.get("REDASH_HSTS_MAX_AGE", talisman.ONE_YEAR_IN_SECS)) +HSTS_INCLUDE_SUBDOMAINS = parse_boolean( + os.environ.get("REDASH_HSTS_INCLUDE_SUBDOMAINS", "false")) + +# Whether and how to send Content-Security-Policy response headers. +# See https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers/Content-Security-Policy +# for more information. +# Overriding this value via an environment variables requires setting it +# as a string in the general CSP format of a semicolon separated list of +# individual CSP directives, see https://github.com/GoogleCloudPlatform/flask-talisman#example-7 +# for more information. E.g.: +CONTENT_SECURITY_POLICY = os.environ.get( + "REDASH_CONTENT_SECURITY_POLICY", + "default-src 'self'; style-src 'self' 'unsafe-inline'; script-src 'self' 'unsafe-eval'; font-src 'self' data:; img-src 'self' http: https: data:; object-src 'none'; frame-ancestors 'none';" +) +CONTENT_SECURITY_POLICY_REPORT_URI = os.environ.get( + "REDASH_CONTENT_SECURITY_POLICY_REPORT_URI", "") +CONTENT_SECURITY_POLICY_REPORT_ONLY = parse_boolean( + os.environ.get("REDASH_CONTENT_SECURITY_POLICY_REPORT_ONLY", "false")) +CONTENT_SECURITY_POLICY_NONCE_IN = array_from_string( + os.environ.get("REDASH_CONTENT_SECURITY_POLICY_NONCE_IN", "")) + +# Whether and how to send Referrer-Policy response headers. Defaults to +# 'strict-origin-when-cross-origin'. +# See https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers/Referrer-Policy +# for more information. +REFERRER_POLICY = os.environ.get( + "REDASH_REFERRER_POLICY", "strict-origin-when-cross-origin") +# Whether and how to send Feature-Policy response headers. Defaults to +# an empty value. +# See https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers/Feature-Policy +# for more information. +FEATURE_POLICY = os.environ.get("REDASH_REFERRER_POLICY", "") + MULTI_ORG = parse_boolean(os.environ.get("REDASH_MULTI_ORG", "false")) GOOGLE_CLIENT_ID = os.environ.get("REDASH_GOOGLE_CLIENT_ID", "") @@ -111,9 +188,6 @@ def all_settings(): STATIC_ASSETS_PATH = fix_assets_path(os.environ.get("REDASH_STATIC_ASSETS_PATH", "../client/dist/")) JOB_EXPIRY_TIME = int(os.environ.get("REDASH_JOB_EXPIRY_TIME", 3600 * 12)) -COOKIE_SECRET = os.environ.get("REDASH_COOKIE_SECRET", "c292a0a3aa32397cdb050e233733900f") -SESSION_COOKIE_SECURE = parse_boolean(os.environ.get("REDASH_SESSION_COOKIE_SECURE") or str(ENFORCE_HTTPS)) -SECRET_KEY = os.environ.get('REDASH_SECRET_KEY', COOKIE_SECRET) LOG_LEVEL = os.environ.get("REDASH_LOG_LEVEL", "INFO") LOG_STDOUT = parse_boolean(os.environ.get('REDASH_LOG_STDOUT', 'false')) diff --git a/requirements.txt b/requirements.txt index b3b0d66840..97067862ee 100644 --- a/requirements.txt +++ b/requirements.txt @@ -15,7 +15,7 @@ requests-oauthlib>=0.6.2,<1.2.0 Flask-SQLAlchemy==2.3.2 Flask-Migrate==2.0.1 flask-mail==0.9.1 -flask-sslify==0.1.5 +flask-talisman==0.6.0 Flask-Limiter==0.9.3 passlib==1.6.2 aniso8601==1.1.0 diff --git a/setup/generate_key.sh b/setup/generate_key.sh index 05eec7f9c7..93b97a0ea6 100644 --- a/setup/generate_key.sh +++ b/setup/generate_key.sh @@ -3,10 +3,12 @@ FLAG="/var/log/generate_secrets.log" if [ ! -f $FLAG ]; then COOKIE_SECRET=$(pwgen -1s 32) + SECRET_KEY=$(pwgen -1s 32) POSTGRES_PASSWORD=$(pwgen -1s 32) REDASH_DATABASE_URL="postgresql:\/\/postgres:$POSTGRES_PASSWORD@postgres\/postgres" sed -i "s/REDASH_COOKIE_SECRET=.*/REDASH_COOKIE_SECRET=$COOKIE_SECRET/g" /opt/redash/env + sed -i "s/REDASH_SECRET_KEY=.*/REDASH_SECRET_KEY=$SECRET_KEY/g" /opt/redash/env sed -i "s/POSTGRES_PASSWORD=.*/POSTGRES_PASSWORD=$POSTGRES_PASSWORD/g" /opt/redash/env sed -i "s/REDASH_DATABASE_URL=.*/REDASH_DATABASE_URL=$REDASH_DATABASE_URL/g" /opt/redash/env diff --git a/setup/setup.sh b/setup/setup.sh index b45cc6772e..388edd674f 100644 --- a/setup/setup.sh +++ b/setup/setup.sh @@ -6,7 +6,7 @@ REDASH_BASE_PATH=/opt/redash install_docker(){ # Install Docker - sudo apt-get update + sudo apt-get update sudo apt-get -yy install apt-transport-https ca-certificates curl software-properties-common wget pwgen curl -fsSL https://download.docker.com/linux/ubuntu/gpg | sudo apt-key add - sudo add-apt-repository "deb [arch=amd64] https://download.docker.com/linux/ubuntu $(lsb_release -cs) stable" @@ -38,6 +38,7 @@ create_config() { fi COOKIE_SECRET=$(pwgen -1s 32) + SECRET_KEY=$(pwgen -1s 32) POSTGRES_PASSWORD=$(pwgen -1s 32) REDASH_DATABASE_URL="postgresql://postgres:${POSTGRES_PASSWORD}@postgres/postgres" @@ -46,6 +47,7 @@ create_config() { echo "REDASH_REDIS_URL=redis://redis:6379/0" >> $REDASH_BASE_PATH/env echo "POSTGRES_PASSWORD=$POSTGRES_PASSWORD" >> $REDASH_BASE_PATH/env echo "REDASH_COOKIE_SECRET=$COOKIE_SECRET" >> $REDASH_BASE_PATH/env + echo "REDASH_SECRET_KEY=$SECRET_KEY" >> $REDASH_BASE_PATH/env echo "REDASH_DATABASE_URL=$REDASH_DATABASE_URL" >> $REDASH_BASE_PATH/env } diff --git a/tests/handlers/test_data_sources.py b/tests/handlers/test_data_sources.py index f07a2b3719..3a5aa0d922 100644 --- a/tests/handlers/test_data_sources.py +++ b/tests/handlers/test_data_sources.py @@ -1,7 +1,7 @@ from funcy import pairwise from tests import BaseTestCase -from redash.models import DataSource, Query +from redash.models import DataSource class TestDataSourceGetSchema(BaseTestCase): diff --git a/tests/handlers/test_embed.py b/tests/handlers/test_embed.py index 18f119d786..80d7b9b907 100644 --- a/tests/handlers/test_embed.py +++ b/tests/handlers/test_embed.py @@ -2,6 +2,15 @@ from redash.models import db +class TestUnembedables(BaseTestCase): + def test_not_embedable(self): + query = self.factory.create_query() + res = self.make_request('get', '/api/queries/{0}'.format(query.id)) + self.assertEquals(res.status_code, 200) + self.assertIn("frame-ancestors 'none'", res.headers['Content-Security-Policy']) + self.assertEqual(res.headers['X-Frame-Options'], 'deny') + + class TestEmbedVisualization(BaseTestCase): def test_sucesss(self): vis = self.factory.create_visualization() @@ -10,6 +19,8 @@ def test_sucesss(self): res = self.make_request("get", "/embed/query/{}/visualization/{}".format(vis.query_rel.id, vis.id), is_json=False) self.assertEqual(res.status_code, 200) + self.assertIn('frame-ancestors *', res.headers['Content-Security-Policy']) + self.assertNotIn("X-Frame-Options", res.headers) # TODO: bring back? # def test_parameters_on_embeds(self): @@ -49,6 +60,8 @@ def test_success(self): res = self.make_request('get', '/public/dashboards/{}'.format(api_key.api_key), user=False, is_json=False) self.assertEqual(res.status_code, 200) + self.assertIn('frame-ancestors *', res.headers['Content-Security-Policy']) + self.assertNotIn("X-Frame-Options", res.headers) def test_works_for_logged_in_user(self): dashboard = self.factory.create_dashboard() @@ -72,6 +85,7 @@ def test_inactive_token(self): # def test_token_doesnt_belong_to_dashboard(self): # pass + class TestAPIPublicDashboard(BaseTestCase): def test_success(self): dashboard = self.factory.create_dashboard() @@ -79,6 +93,8 @@ def test_success(self): res = self.make_request('get', '/api/dashboards/public/{}'.format(api_key.api_key), user=False, is_json=False) self.assertEqual(res.status_code, 200) + self.assertIn('frame-ancestors *', res.headers['Content-Security-Policy']) + self.assertNotIn("X-Frame-Options", res.headers) def test_works_for_logged_in_user(self): dashboard = self.factory.create_dashboard() diff --git a/tests/handlers/test_favorites.py b/tests/handlers/test_favorites.py index 8d704fd1c7..4253edb7bc 100644 --- a/tests/handlers/test_favorites.py +++ b/tests/handlers/test_favorites.py @@ -1,6 +1,5 @@ from tests import BaseTestCase -from redash import models -from redash.models import db + class TestQueryFavoriteResource(BaseTestCase): def test_favorite(self): diff --git a/tests/handlers/test_queries.py b/tests/handlers/test_queries.py index bda5b423e9..d45761d340 100644 --- a/tests/handlers/test_queries.py +++ b/tests/handlers/test_queries.py @@ -19,8 +19,7 @@ def test_get_query(self): self.assertResponseEqual(expected, rv.json) def test_get_all_queries(self): - queries = [self.factory.create_query() for _ in range(10)] - + [self.factory.create_query() for _ in range(10)] rv = self.make_request('get', '/api/queries') self.assertEquals(rv.status_code, 200) diff --git a/tests/handlers/test_settings.py b/tests/handlers/test_settings.py index 4c73b1d253..e8c9606e04 100644 --- a/tests/handlers/test_settings.py +++ b/tests/handlers/test_settings.py @@ -13,14 +13,14 @@ def test_post(self): updated_org = Organization.get_by_slug(self.factory.org.slug) self.assertEqual(rv.json['settings']['auth_password_login_enabled'], True) self.assertEqual(updated_org.settings['settings']['auth_password_login_enabled'], True) - + def test_updates_google_apps_domains(self): admin = self.factory.create_admin() domains = ['example.com'] rv = self.make_request('post', '/api/settings/organization', data={'auth_google_apps_domains': domains}, user=admin) updated_org = Organization.get_by_slug(self.factory.org.slug) self.assertEqual(updated_org.google_apps_domains, domains) - + def test_get_returns_google_appas_domains(self): admin = self.factory.create_admin() domains = ['example.com'] @@ -28,4 +28,3 @@ def test_get_returns_google_appas_domains(self): rv = self.make_request('get', '/api/settings/organization', user=admin) self.assertEqual(rv.json['settings']['auth_google_apps_domains'], domains) - From 6c26aa7a997e8a6aa968ea743bc716bf20617475 Mon Sep 17 00:00:00 2001 From: Jannis Leidel Date: Wed, 27 Mar 2019 16:26:00 +0100 Subject: [PATCH 006/179] Render LDAP and remote auth login links correctly when multi org mode is enabled. (#3530) * Make LDAP auth handler org scoped. * Render LDAP and remote auth login links correctly when multi org mode is enabled. --- redash/authentication/ldap_auth.py | 14 ++++++++------ redash/templates/login.html | 4 ++-- tests/test_handlers.py | 25 ++++++++++++++++--------- 3 files changed, 26 insertions(+), 17 deletions(-) diff --git a/redash/authentication/ldap_auth.py b/redash/authentication/ldap_auth.py index 39fb38cb92..9bde870945 100644 --- a/redash/authentication/ldap_auth.py +++ b/redash/authentication/ldap_auth.py @@ -1,26 +1,28 @@ import logging -logger = logging.getLogger('ldap_auth') +import sys from redash import settings from flask import flash, redirect, render_template, request, url_for, Blueprint -from flask_login import current_user, login_required, login_user, logout_user +from flask_login import current_user try: - from ldap3 import Server, Connection, SIMPLE, ANONYMOUS, NTLM + from ldap3 import Server, Connection except ImportError: if settings.LDAP_LOGIN_ENABLED: - logger.error("The ldap3 library was not found. This is required to use LDAP authentication (see requirements.txt).") - exit() + sys.exit("The ldap3 library was not found. This is required to use LDAP authentication (see requirements.txt).") from redash.authentication import create_and_login_user, logout_and_redirect_to_index, get_next_path from redash.authentication.org_resolving import current_org +from redash.handlers.base import org_scoped_rule + +logger = logging.getLogger('ldap_auth') blueprint = Blueprint('ldap_auth', __name__) -@blueprint.route("/ldap/login", methods=['GET', 'POST']) +@blueprint.route(org_scoped_rule("/ldap/login"), methods=['GET', 'POST']) def login(org_slug=None): index_url = url_for("redash.index", org_slug=org_slug) unsafe_next_path = request.args.get('next', index_url) diff --git a/redash/templates/login.html b/redash/templates/login.html index 27248493e3..71bc8b4aec 100644 --- a/redash/templates/login.html +++ b/redash/templates/login.html @@ -25,11 +25,11 @@ {% endif %} {% if show_remote_user_login %} - + {% endif %} {% if show_ldap_login %} - + {% endif %} {% if show_password_login %} diff --git a/tests/test_handlers.py b/tests/test_handlers.py index 8c1b00e9e7..04d93a55cb 100644 --- a/tests/test_handlers.py +++ b/tests/test_handlers.py @@ -1,4 +1,3 @@ -from flask import url_for from flask_login import current_user from funcy import project from mock import patch @@ -91,18 +90,26 @@ def setUp(self): super(TestLogin, self).setUp() self.factory.org.set_setting('auth_password_login_enabled', True) - @classmethod - def setUpClass(cls): - settings.ORG_RESOLVING = "single_org" - - @classmethod - def tearDownClass(cls): - settings.ORG_RESOLVING = "multi_org" - def test_get_login_form(self): rv = self.client.get('/default/login') self.assertEquals(rv.status_code, 200) + def test_get_login_form_remote_auth(self): + """Make sure the remote auth link can be rendered correctly on the + login page when the remote user login feature is enabled""" + old_remote_user_enabled = settings.REMOTE_USER_LOGIN_ENABLED + old_ldap_login_enabled = settings.LDAP_LOGIN_ENABLED + try: + settings.REMOTE_USER_LOGIN_ENABLED = True + settings.LDAP_LOGIN_ENABLED = True + rv = self.client.get('/default/login') + self.assertEquals(rv.status_code, 200) + self.assertIn('/{}/remote_user/login'.format(self.factory.org.slug), rv.data) + self.assertIn('/{}/ldap/login'.format(self.factory.org.slug), rv.data) + finally: + settings.REMOTE_USER_LOGIN_ENABLED = old_remote_user_enabled + settings.LDAP_LOGIN_ENABLED = old_ldap_login_enabled + def test_submit_non_existing_user(self): with patch('redash.handlers.authentication.login_user') as login_user_mock: rv = self.client.post('/default/login', data={'email': 'arik', 'password': 'password'}) From b5d97e25b7cbf1632f1a6c7a9b85abb1861eba1f Mon Sep 17 00:00:00 2001 From: Ran Byron Date: Wed, 27 Mar 2019 17:47:12 +0200 Subject: [PATCH 007/179] Browser support config (#3609) * Browser support config * Removed some offending code * Added unsupported html page and redirect for IE * Typo in regex * Made html page static * Added redirect script to multi_org * Moved static html page to client/app --- client/.babelrc | 5 ++- client/.eslintrc.js | 3 +- client/app/index.html | 12 +++++ client/app/multi_org.html | 12 +++++ client/app/unsupported.html | 61 +++++++++++++++++++++++++ package-lock.json | 90 +++++++++++++++++++++++++++++-------- package.json | 18 +++++++- webpack.config.js | 1 + 8 files changed, 180 insertions(+), 22 deletions(-) create mode 100644 client/app/unsupported.html diff --git a/client/.babelrc b/client/.babelrc index 7ba3d2057b..4de7bd4792 100644 --- a/client/.babelrc +++ b/client/.babelrc @@ -1,7 +1,10 @@ { "presets": [ ["@babel/preset-env", { - "targets": "> 0.5%, last 2 versions, Firefox ESR, ie 11, not dead", + "exclude": [ + "@babel/plugin-transform-async-to-generator", + "@babel/plugin-transform-arrow-functions" + ], "useBuiltIns": "usage" }], "@babel/preset-react" diff --git a/client/.eslintrc.js b/client/.eslintrc.js index 083d993905..410455db7b 100644 --- a/client/.eslintrc.js +++ b/client/.eslintrc.js @@ -1,6 +1,7 @@ module.exports = { root: true, - extends: ["airbnb"], + extends: ["airbnb", "plugin:compat/recommended"], + plugins: ["jest", "compat"], settings: { "import/resolver": "webpack" }, diff --git a/client/app/index.html b/client/app/index.html index 9909b295d5..8e5308b773 100644 --- a/client/app/index.html +++ b/client/app/index.html @@ -5,6 +5,18 @@ Redash + + diff --git a/client/app/multi_org.html b/client/app/multi_org.html index 00a7c8a3c4..39da9818de 100644 --- a/client/app/multi_org.html +++ b/client/app/multi_org.html @@ -5,6 +5,18 @@ Redash + + diff --git a/client/app/unsupported.html b/client/app/unsupported.html new file mode 100644 index 0000000000..2e6fac7fad --- /dev/null +++ b/client/app/unsupported.html @@ -0,0 +1,61 @@ + + + + Redash doesn't support your browser + + + + +
+ +

Whoops... Redash doesn't support your browser

+
+
+

Download one of these free and up-to-date browsers:

+ +
+ + \ No newline at end of file diff --git a/package-lock.json b/package-lock.json index c2f8c28137..cf86019e9b 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1862,6 +1862,12 @@ "integrity": "sha1-WWZ/QfrdTyDMvCu5a41Pf3jsA2c=", "dev": true }, + "ast-metadata-inferer": { + "version": "0.1.1", + "resolved": "https://registry.npmjs.org/ast-metadata-inferer/-/ast-metadata-inferer-0.1.1.tgz", + "integrity": "sha512-hc9w8Qrgg9Lf9iFcZVhNjUnhrd2BBpTlyCnegPVvCe6O0yMrF57a6Cmh7k+xUsfUOMh9wajOL5AsGOBNEyTCcw==", + "dev": true + }, "ast-types-flow": { "version": "0.0.7", "resolved": "https://registry.npmjs.org/ast-types-flow/-/ast-types-flow-0.0.7.tgz", @@ -2479,7 +2485,7 @@ }, "bl": { "version": "1.2.2", - "resolved": "http://registry.npmjs.org/bl/-/bl-1.2.2.tgz", + "resolved": "https://registry.npmjs.org/bl/-/bl-1.2.2.tgz", "integrity": "sha512-e8tQYnZodmebYDWGH7KMRvtzKXaJHx3BbilrgZCfvyLUYdKpK1t5PSPmpkny/SgiTSCnjfLW7v5rlONXVFkQEA==", "requires": { "readable-stream": "^2.3.5", @@ -2493,7 +2499,7 @@ }, "readable-stream": { "version": "2.3.6", - "resolved": "http://registry.npmjs.org/readable-stream/-/readable-stream-2.3.6.tgz", + "resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-2.3.6.tgz", "integrity": "sha512-tQtKA9WIAhBF3+VLAseyMqZeBjW0AHJoxOtYqSUZNJxauErmLbVm2FW1y+J/YA9dUrAC39ITejlZWhVIwawkKw==", "requires": { "core-util-is": "~1.0.0", @@ -2705,7 +2711,7 @@ }, "readable-stream": { "version": "2.3.6", - "resolved": "http://registry.npmjs.org/readable-stream/-/readable-stream-2.3.6.tgz", + "resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-2.3.6.tgz", "integrity": "sha512-tQtKA9WIAhBF3+VLAseyMqZeBjW0AHJoxOtYqSUZNJxauErmLbVm2FW1y+J/YA9dUrAC39ITejlZWhVIwawkKw==", "requires": { "core-util-is": "~1.0.0", @@ -5517,6 +5523,43 @@ "integrity": "sha512-hkpLN7VVoGGsofZjUhcQ+sufC3FgqMJwD0DvAcRfxY1tVRyQyVsqpaKnToPHJQOrRo0FQ0fSEDwW2gr4rsNdGA==", "dev": true }, + "eslint-plugin-compat": { + "version": "3.0.1", + "resolved": "https://registry.npmjs.org/eslint-plugin-compat/-/eslint-plugin-compat-3.0.1.tgz", + "integrity": "sha512-i0hx+Fm2A2rn35xtCpRAdg1tawe+Gsv8ydR3s6UMINGy9HzW1Yw7ojnm9E/bSznWjdIatKN/cVAWbG/uy4Munw==", + "dev": true, + "requires": { + "@babel/runtime": "^7.3.4", + "ast-metadata-inferer": "^0.1.1", + "browserslist": "^4.4.2", + "caniuse-db": "^1.0.30000947", + "mdn-browser-compat-data": "^0.0.70", + "semver": "^5.6.0" + }, + "dependencies": { + "@babel/runtime": { + "version": "7.3.4", + "resolved": "https://registry.npmjs.org/@babel/runtime/-/runtime-7.3.4.tgz", + "integrity": "sha512-IvfvnMdSaLBateu0jfsYIpZTxAc2cKEXEMiezGGN75QcBcecDUKd3PgLAncT0oOgxKy8dd8hrJKj9MfzgfZd6g==", + "dev": true, + "requires": { + "regenerator-runtime": "^0.12.0" + } + }, + "caniuse-db": { + "version": "1.0.30000950", + "resolved": "https://registry.npmjs.org/caniuse-db/-/caniuse-db-1.0.30000950.tgz", + "integrity": "sha512-mS/KbErOeYOVF8W4GdMmHAyNm9p4XGXLP4Nde75b1uadwzUr+dFd0RCVheQIB8+yKs1PGW73Bnejch0G1s0FvQ==", + "dev": true + }, + "regenerator-runtime": { + "version": "0.12.1", + "resolved": "https://registry.npmjs.org/regenerator-runtime/-/regenerator-runtime-0.12.1.tgz", + "integrity": "sha512-odxIc1/vDlo4iZcfXqRYFj0vpXFNoGdKMAUieAlFYO6m/nl5e9KR/beGf41z4a1FI+aQgtjhuaSlDxQ0hmkrHg==", + "dev": true + } + } + }, "eslint-plugin-cypress": { "version": "2.2.1", "resolved": "https://registry.npmjs.org/eslint-plugin-cypress/-/eslint-plugin-cypress-2.2.1.tgz", @@ -6251,7 +6294,7 @@ }, "finalhandler": { "version": "1.1.1", - "resolved": "http://registry.npmjs.org/finalhandler/-/finalhandler-1.1.1.tgz", + "resolved": "https://registry.npmjs.org/finalhandler/-/finalhandler-1.1.1.tgz", "integrity": "sha512-Y1GUDo39ez4aHAw7MysnUD5JzYX+WaIj8I57kO3aEPT1fFRL4sr7mjei97FgnwhAyyzRYmQZaTHb2+9uZ1dPtg==", "dev": true, "requires": { @@ -8738,7 +8781,7 @@ }, "readable-stream": { "version": "2.3.6", - "resolved": "http://registry.npmjs.org/readable-stream/-/readable-stream-2.3.6.tgz", + "resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-2.3.6.tgz", "integrity": "sha512-tQtKA9WIAhBF3+VLAseyMqZeBjW0AHJoxOtYqSUZNJxauErmLbVm2FW1y+J/YA9dUrAC39ITejlZWhVIwawkKw==", "requires": { "core-util-is": "~1.0.0", @@ -9080,7 +9123,7 @@ }, "readable-stream": { "version": "2.3.6", - "resolved": "http://registry.npmjs.org/readable-stream/-/readable-stream-2.3.6.tgz", + "resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-2.3.6.tgz", "integrity": "sha512-tQtKA9WIAhBF3+VLAseyMqZeBjW0AHJoxOtYqSUZNJxauErmLbVm2FW1y+J/YA9dUrAC39ITejlZWhVIwawkKw==", "dev": true, "requires": { @@ -9174,7 +9217,7 @@ }, "html-webpack-plugin": { "version": "3.2.0", - "resolved": "http://registry.npmjs.org/html-webpack-plugin/-/html-webpack-plugin-3.2.0.tgz", + "resolved": "https://registry.npmjs.org/html-webpack-plugin/-/html-webpack-plugin-3.2.0.tgz", "integrity": "sha1-sBq71yOsqqeze2r0SS69oD2d03s=", "dev": true, "requires": { @@ -9821,7 +9864,7 @@ }, "is-obj": { "version": "1.0.1", - "resolved": "http://registry.npmjs.org/is-obj/-/is-obj-1.0.1.tgz", + "resolved": "https://registry.npmjs.org/is-obj/-/is-obj-1.0.1.tgz", "integrity": "sha1-PkcprB9f3gJc19g6iW2rn09n2w8=" }, "is-observable": { @@ -11374,7 +11417,7 @@ }, "magic-string": { "version": "0.22.5", - "resolved": "http://registry.npmjs.org/magic-string/-/magic-string-0.22.5.tgz", + "resolved": "https://registry.npmjs.org/magic-string/-/magic-string-0.22.5.tgz", "integrity": "sha512-oreip9rJZkzvA8Qzk9HFs8fZGF/u7H/gtrE8EN6RjKJ9kh2HlC+yQ2QezifqTZfGyiuAV0dRv5a+y/8gBb1m9w==", "requires": { "vlq": "^0.2.2" @@ -11486,12 +11529,12 @@ }, "minimist": { "version": "0.0.8", - "resolved": "http://registry.npmjs.org/minimist/-/minimist-0.0.8.tgz", + "resolved": "https://registry.npmjs.org/minimist/-/minimist-0.0.8.tgz", "integrity": "sha1-hX/Kv8M5fSYluCKCYuhqp6ARsF0=" }, "readable-stream": { "version": "2.3.6", - "resolved": "http://registry.npmjs.org/readable-stream/-/readable-stream-2.3.6.tgz", + "resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-2.3.6.tgz", "integrity": "sha512-tQtKA9WIAhBF3+VLAseyMqZeBjW0AHJoxOtYqSUZNJxauErmLbVm2FW1y+J/YA9dUrAC39ITejlZWhVIwawkKw==", "requires": { "core-util-is": "~1.0.0", @@ -11614,6 +11657,15 @@ "safe-buffer": "^5.1.2" } }, + "mdn-browser-compat-data": { + "version": "0.0.70", + "resolved": "https://registry.npmjs.org/mdn-browser-compat-data/-/mdn-browser-compat-data-0.0.70.tgz", + "integrity": "sha512-++gDpMmrHmZVIFTK9x3z7UjSAM3MYEwZe38yzNejVQe6OUHiLgRvlMW6TQznpIUGuWLesLGUGOg6iyj6JcTB2g==", + "dev": true, + "requires": { + "extend": "3.0.2" + } + }, "media-typer": { "version": "0.3.0", "resolved": "https://registry.npmjs.org/media-typer/-/media-typer-0.3.0.tgz", @@ -11681,7 +11733,7 @@ }, "readable-stream": { "version": "2.3.6", - "resolved": "http://registry.npmjs.org/readable-stream/-/readable-stream-2.3.6.tgz", + "resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-2.3.6.tgz", "integrity": "sha512-tQtKA9WIAhBF3+VLAseyMqZeBjW0AHJoxOtYqSUZNJxauErmLbVm2FW1y+J/YA9dUrAC39ITejlZWhVIwawkKw==", "dev": true, "requires": { @@ -11877,7 +11929,7 @@ }, "readable-stream": { "version": "2.3.6", - "resolved": "http://registry.npmjs.org/readable-stream/-/readable-stream-2.3.6.tgz", + "resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-2.3.6.tgz", "integrity": "sha512-tQtKA9WIAhBF3+VLAseyMqZeBjW0AHJoxOtYqSUZNJxauErmLbVm2FW1y+J/YA9dUrAC39ITejlZWhVIwawkKw==", "dev": true, "requires": { @@ -12370,7 +12422,7 @@ }, "readable-stream": { "version": "2.3.6", - "resolved": "http://registry.npmjs.org/readable-stream/-/readable-stream-2.3.6.tgz", + "resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-2.3.6.tgz", "integrity": "sha512-tQtKA9WIAhBF3+VLAseyMqZeBjW0AHJoxOtYqSUZNJxauErmLbVm2FW1y+J/YA9dUrAC39ITejlZWhVIwawkKw==", "dev": true, "requires": { @@ -12772,7 +12824,7 @@ "dependencies": { "minimist": { "version": "0.0.10", - "resolved": "http://registry.npmjs.org/minimist/-/minimist-0.0.10.tgz", + "resolved": "https://registry.npmjs.org/minimist/-/minimist-0.0.10.tgz", "integrity": "sha1-3j+YVD2/lggr5IrRoMfNqDYwHc8=", "dev": true }, @@ -15920,7 +15972,7 @@ "dependencies": { "minimist": { "version": "0.0.5", - "resolved": "http://registry.npmjs.org/minimist/-/minimist-0.0.5.tgz", + "resolved": "https://registry.npmjs.org/minimist/-/minimist-0.0.5.tgz", "integrity": "sha1-16oye87PUY+RBqxrjwA/o7zqhWY=" } } @@ -16408,7 +16460,7 @@ }, "split": { "version": "0.2.10", - "resolved": "http://registry.npmjs.org/split/-/split-0.2.10.tgz", + "resolved": "https://registry.npmjs.org/split/-/split-0.2.10.tgz", "integrity": "sha1-Zwl8YB1pfOE2j0GPBs0gHPBSGlc=", "requires": { "through": "2" @@ -16766,7 +16818,7 @@ "dependencies": { "readable-stream": { "version": "1.1.14", - "resolved": "http://registry.npmjs.org/readable-stream/-/readable-stream-1.1.14.tgz", + "resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-1.1.14.tgz", "integrity": "sha1-fPTFTvZI44EwhMY23SB54WbAgdk=", "requires": { "core-util-is": "~1.0.0", @@ -16900,7 +16952,7 @@ }, "strip-ansi": { "version": "3.0.1", - "resolved": "http://registry.npmjs.org/strip-ansi/-/strip-ansi-3.0.1.tgz", + "resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-3.0.1.tgz", "integrity": "sha1-ajhfuIU9lS1f8F0Oiq+UJ43GPc8=", "dev": true, "requires": { diff --git a/package.json b/package.json index 9f45fb0afb..53dd04f0ad 100644 --- a/package.json +++ b/package.json @@ -104,6 +104,7 @@ "eslint-import-resolver-webpack": "^0.8.3", "eslint-loader": "^2.1.1", "eslint-plugin-chai-friendly": "^0.4.1", + "eslint-plugin-compat": "^3.0.1", "eslint-plugin-cypress": "^2.0.1", "eslint-plugin-import": "^2.14.0", "eslint-plugin-jest": "^22.2.2", @@ -162,5 +163,20 @@ "npm run test -- --bail --findRelatedTests" ] } - } + }, + "//": "browserslist set to 'Async functions' compatibility", + "browserslist": [ + "Edge >= 15", + "Firefox >= 52", + "Chrome >= 55", + "Safari >= 10.1", + "iOS >= 10.3", + "Opera >= 42", + "op_mob >= 46", + "android >= 67", + "and_chr >= 71", + "and_ff >= 64", + "and_uc >= 11.8", + "samsung >= 6.2" + ] } diff --git a/webpack.config.js b/webpack.config.js index 7d9650c09d..e782dbedbb 100644 --- a/webpack.config.js +++ b/webpack.config.js @@ -72,6 +72,7 @@ const config = { }), new CopyWebpackPlugin([ { from: "client/app/assets/robots.txt" }, + { from: "client/app/unsupported.html" }, { from: "client/app/assets/css/*.css", to: "styles/", flatten: true }, { from: "node_modules/jquery/dist/jquery.min.js", to: "js/jquery.min.js" } ]) From 71afc99ec3988f8ec09101fc1ab16a4948e8eeec Mon Sep 17 00:00:00 2001 From: Byunghwa Yun Date: Thu, 28 Mar 2019 00:48:49 +0900 Subject: [PATCH 008/179] Add phoenix query runner. (#3153) * Add phoenix query runner. * Improved error handling. --- client/app/assets/images/db-logos/phoenix.png | Bin 0 -> 20743 bytes redash/query_runner/phoenix.py | 121 ++++++++++++++++++ redash/settings/__init__.py | 1 + requirements_all_ds.txt | 1 + 4 files changed, 123 insertions(+) create mode 100644 client/app/assets/images/db-logos/phoenix.png create mode 100644 redash/query_runner/phoenix.py diff --git a/client/app/assets/images/db-logos/phoenix.png b/client/app/assets/images/db-logos/phoenix.png new file mode 100644 index 0000000000000000000000000000000000000000..eb56de298ecc41619f1fb0053bc8e8f0cdbf6882 GIT binary patch literal 20743 zcmcdy_aoK+|9#$jT`PNUqEL27=Dju%A$t~?8AZsv$jmAwd+)u;yiq8XO*WCuHM8&i z-1qm7_}*Xc{o!@*eLf$r^L&hR&MW$%jygFh3n>7ATvJ0u9{>pcBLol=;@>P@l-L9C z@Rz2F;v?VbolK&DN0T3LoOdmiP~U&Fopkcqg^~B(wUJ5T_QOQ+`iT&Z%J$#bcVqrp9V@2jUc7Dfa!QNfUDdRChcU_O%fY~>t4>VAM}(rSU4v_{UaPVE zj9m-R{dAJm^IZG?^(9qxJH9Q_YCF^B$tKb-Yk9X4HE>X=vu*kK$tH+!KR*2^9kd6h zcwk3m^Q85S;oHZjk{O~!X*>(w&u=_`t6cvx`!R-&c66>#XV9GN;?A0Hdjt=nboC3v z2LXdQHmvX=R$JT0sl`1gkVEOpS;u#NfW$dg!0;-gL?46W;FCjQ9Kw;jMQth*-?6e5 z&#PG|hoBn7k|6X`aGz9=C?awvC&hdlP6VWd}Ln7jQucgeaby>m73y~tv`<4Gs4zyLKNe6 z`+n92e&wm^6aD$L^9>uzN~`+bm)+sG1<#ku#_i{o7C)o&nLtX4The~|JmOKbj>I!Q z=R<`&wUedEE{V{sCy+1Wab5nVspSHt(cTQ>X*xGvNiIuu1S_3jyILm}Y@R&%{#Lu1 z)gszk!i}J3A?Dpdk_tISG5b@p@m-8;A3fcK*Y)7ImqLavyO8Cswas34fQ)w@4{ne7 zZ)Y=(mmyy-Gk)LC>z$yLtRvABqJpS+?;lp#r`XrV&to=uvAmF8%&yYAUH{K$%%`UP zNXcbJk@msk9oIlPUfv?}oU#}4UMC`S5d;j&Ds#6*&+m_n)R4F1C{{?^?s~60wZ>|V zlR5jvER^gxF$Eio?rp(Pc5LToO#eZ)dHR$*zf&&PXJJw9QfK*vBDrCXT?>uocVWf2 zZ6u;CX1|zhVmyp3c1@9EA+tYi-uUD747po#MDSp zIH-`Qhzzb)cBun7NgU^%mb0(^xQGFiK!QdCp+^`sif;55(fSfX_O$1VUHv>{WBfPLIk_Djxq@cj; zSSp@ivMwSUe>eM2AMmstJw&Xq^ZMrH69AA511Ka5&|ki-NM5DVCACGcx?Ci4zVcyz z7dp&(o9Y%=Ne8G3DFW)?LF`8{u_?#Pj}iAU{qG^NB(eX$^zZ3Raw z1WZs#FA7T{jPrEuBXA5D+%rpS;)TO#l|vjW95r=V9YLG*-N+VyC|V(faFgY$hpAhb0O=Ab?4HG{-$_T-Sa@`FDS2UxU)VnPAkRvVX)J=i zwM}fE21D74pc?4Bo`_2jg>bEdl(@gGHxqFpUxWKmk@h*5ziYCyfpA^o_O4>7w;lK& zJzTLSmpvc&VwbQM9J1W{is%~7@+xOuyN49@Xnnr8BH`Ndo3|IA9XX%DfcSHLGv2b0 zua%Um1?|D^b>&Sp?^(K5Hek2D{DdIOM%%Dic1sA_B=Y$M@rQEDuZVFDcN5YMuyP7! z0;o0No_9W6Ot^eP(2LsFb!3N}GAxcC`B;q^L-YkeRp*Ls^gkGq#EBXp`pz%^J6Ool zxqz=0}X56ScUVK`9L%>2<^&LMO0rXfG^cji&vIUSENrsE;VvUKpd(OxqGTn36qoR zeYd6{I=c-LUDifmiYpHUkopOFfCNsPyhV+v4?2fUAX2NcLEu-B#U_C zM4AXe4SUE_WOcFeB^~0=7SWCr=RiR08~Cr<&L^iGO3tS$PLv>)VTtoGuzBJfx!+~X zEw_06-o_@Mits(QUu0qT8F&2>5$RyB2?7>;`+pWwNy@z`nSXWpUK@qxyKX; z;od2Mv>15e`qcyq{<{aFC)Ll1vkfke4J@zvucdzSa(1wBxFa+~+x-u^H+|ElhM260 zb5kCpKd@KR?A>TQ)?GgFEQbveEDSu+0n z!<*BbRu3r7jSshA18349G4CK#u?K#|f2WtOEEnHehTmXQw$WDc=rVT9n3CF+>e#%g zV^SnJyH04uN@fTH-=V+hw() zUrawm!!_zkS}de?F>}`Gih0?>O)p3VrO4TqcDMo zaxI~kLMDn?hazg?mDejCi%MT0i^=LhcHzKtJ(Z8g{pCg}jifl~19sQn_C8lbI;5b` z7>Y)4X#E_8fTWCT8<-KtegaYzu&UWnYcwF@E=tsX>lGG>yQPsT8S1ft=KZLRrTY4t{nS^P*4wS4X zC9Ril%6v_kr-2em-I=Kwu#UNM$kZn>>vaOoPs!q)}3CARa^|u%xSRBF0V5!Q?Q-* zJ8ssu1k@-fHLzgf0+VJH5dyU=X5sIL?zb3mNjIw#+i+E|P~wcIfHL>>Rb`km3bs zZx=i^kk(@C!u_@RHNXu33*@!&=SG?;dJmzV`;OAd%38JWO@vFn(dV(4yj~#xvGy@N zSod0C+|^MpQKy0Vq#<`C9i8*RYneZ$OG$`aV(9d$Oo)6~)x}j`?`A zyqOx7Ia0Ntd{B~ebr{WJR#pgo;UT6oAwoNLev^t?kZa+UyM5=feb(9=X4P+F&*z1I z)sFQy@Uq$bP%JL7;-~4gd%c|*dx;Bgp$8pDmH|@$P$TQ2p*GO7LQ__;zJRtEEymYu z5&Miim}x3V%f0<;WI((5(Q4#Qz)fSSsknv2l6?$PsFvL%&ea_Ol(dcnT8Onm;3k^F z;f^*6(wrG_#*^B#5o157I6OY)D`FDXXD-4a7E*qu9k{Otcjm9|nJuRxt-RuRMk~}Z zNpWq7x|3@mdM7mIp{j{9vYIVLD%KxrR2XMpRn_FA(auwDx$M@pnG*t)mK}Fs!!HDV zQu?-uFuTCto)jhW+=5XE%1gmxDOrdk?@W%|}#kQy0r+||P_=X5?-_DG& zCVjs=dG+@mn-JsJYN^e$i6bnUg1O)ZXyN4)Y;kHlw zwsPSTUxt9K|L{j1-JD5oe-8sFpqhKi6dVtsN>k^7Ogpm*G)&ps@ETqIx3)=%(-R3n zqM~j?ib=bSSY2vI4(#q>7?IC|4qhHy_0nt1_{&@4k?v4H13+_F`!o5=q=RX;>FTxV zM4ST*1)5KqW59befO4E9cZ)O_WAULsWAK-M2f;`fgVk^gub=mRftFw{&A$oTGus)79BegW{1J76FP z)NpepJ&^AFG_h;m_g9rNKd`pvGJS-kqKq$U+Ls{qLenZ9vs}1tbwl+8wU5Xx20l=* z&M!P2@g}R=B~^H24@Wz3Kd9`qZ~w4Cs3Y@wL2>T~2asb@J?&5hP`Ow4l7-;?P=Lu- zA&GrGQ>(OGxU`-t2u_2#d+BnFI0+A=2?6g+d(aWT8Jq0)uF|ZV&IdP)mLt#XF1g&| z*%y=OKyBa=RJr&J$Bu^*?t`f77wPbpz@#Ax7a7~*^#H%dkxdiwVG7@s#&_Klge z{QN(;)h=I7rs>!=Qj;OP2^zn|@4XlmtKeO7&YgVRg5jD~@4@jU2Jk`92=amBQTRB5 z`O22?w|?NMQ=7XKUuPk+ml^@SvwAd~gbldC3Q9i9EV@9OPo4ssSSm51mkb9Qg*;zYp|ZMDX!uw9cBn`?;_*UU{iHop(=}KTU?0cSR7RSx9_g@ z?}>2_q~b0+*X;txLFOYE@dh=p48s}YZX5{>`aTv43@m-=OA$v9=CrR*Ijy-cQ4Yze zaRp-@D|64l9uwGO0ehPu(Uzx?AQioXpM8SCDJG@o9+8bQL5>J4CRJHIC8bJa zt++YlwlE^-`&rTXbHz&8xUt`2*q^_t*BE&shglXNYx&s9S30dre9tUG=3WK(QsO+k zC~;cHiO6dq-(+vX_qNJ`=1Zen;mG4~^@D~9tLh@awOiH5aKsM|H0$(;&?0v{|1|+5 z`6wx<7q2;a4AA^~$ML+FiP%v}2~#QhDNC5B47BTvuD@uY)1(ipGK6N z4-9|WEcK&slXMha&Hi!+g0B&Q*Dc{x!I@4qYUGQVMqs^yb&VeQ4na|Rr8wW^M^^Es*O+MO9}z@#h-m$?!ipX9(p?#+#3s1AHM>ritNoK zA&^Mpb{=2zSLg`9}CW35q^GHgOkueiac`J67CuWCo~+z}Wl5uFfyd6R@~pujpR0*|k{~Lmq&d&s^F*WGdV< z!x%^PoOEa67abtw&0oSO5FkD@vGv2BT$uq|_<>IfAmid#tV>KDiO3v>DNn%Ugem0H zYZxdVfQDhFF>x}1#$^C_MjasfSK$7A<1ashPVRa)opDobVe2yNiK{mR5f;5apj5$3wXKBi|`z{&3Rw1KUw z^Oqk}KMTYTg7V={z6Ano!(B_KV_wxnF@Bf-s#5e;`9c<&ni{xmSBvS30Uqjgw42hEipSK=A^wo#n zWMtg?219I#@$NS;LzvuT2FeA09B0A zU;+2PxTke}Ok3%pC*5n?wdp;0Yy`+AAVb7pFZL@g+4NySQn=;KKVPHBA1Kb<;-U^$ zXm+OfX(s^D_EC}NYGB66eR{*9utWCDK$(gd$8$)sL;Su)l>T^N_39Rt<28&n|8Nn< zypbBLIg<TXk%OLVOj!OZC*1+zsS-3&w$d>&gz|AAO;-xDPGl zo)E!suNR7uv9MV z8JWW>ZnepZ9o%ufw)%p~_~mmym~JZ{IJe6a0x7Pa$$)Q!%}ogwth`m!MC`CJeZ>{A zg=%sU*Ij_FfT3M3Das+DbIf1S$K8p0_61)Y$x*-HH6$SJ+QvCzs*u@n%dKlV66+dI zmzw_Mo|d`DUVLA*3ADj&Ye$vN`9~1n>k*9%JHHVDKVlODl$8_k{W?((akNH9kbsQn zKTi%n6>*fm&ZwxT2403d6|N)3p_+VZiYD*Hi=B|outu$ldYGwq97Xo@L*uk%l zo5kGH2kvyve3g!{S~)6|^XvN5*c{R0kbZQ0kSxHB%2IRL>r^RQyZ+vrRM*~b=Mm#| zPi8SzaBglEme5TDN1Z%Bx^);R_IO3O_gWk=x{3~+LL>M^cdiih+{gasGHe7S zh(>7dw7Tdxt7yJ>P4m(BiN%+6fOg--c#P+CBy#TS4QhUdjfF?k+0b=8Ft_ZHI%=NN z(Z7iw4`{*X#w#k2)0(9Qb_h_z3>LXUpF8T$Vg#4Ep0-ujRHungqs(9UMuASN^iRpz z%n&*nY3MD~wI!_vGj}B}d3IuPm`&|fnol-+#kQ+bt@{XFwnGY!gRO=T8A*iDUOk!ENIK zNj6I!+T#qoZhq=cKey|j%5zu`7F`-$=^+LHRsQ-0D}l^SrojGlzmb{HuOdS?rFrhyJl6rytV-I?mbdt!DuVo(kQ0_-&GbH6>S0r0hq? z=L>-bLMlt-qlFA3tJHxC-%(0ACbTsG0)jwPA9guhA&L66VHs$qSVi!{3HrDbi59$%!MI+;YRuR_R}DvrsgOJfohdCCHq?o58l3yhxVZVCNcD&gSxW>qJR##5Es^7p z6gL-gt_FB!-UaC1fuo-YjxObVK3j25zMgH`jAbE5C?%F-L?Je&@Dk@g=a!#HZQ=kw zi20nvtpT|ozN*!-BKBtAB{-~UKD2A!yQRE@oo7M@CQAnJIZzVB;`5kE3-6-cp`Zq= z=+Mvz40RCha(}LjyRik+L&*b;3A0v$dOCcf9|zs(XTix_P~v7!J}!2O={hnp$K1U~ ztn-Lt#V)jTh6J7W^A*%tee$2h+Vdu7AU_g_l|M?PNJu(V%oqofH7mhM^UA_a5Wa%gd~^R6c;^k!78Di3zu*Nd`J%+4b9Xkf zd~Hon_Kc2BRx@{INXh1{qtTVY5T}Emu5Et}W#mL&X(M*uC^+3}P5p_S2){!KeG1JP zA^>lB3iqz=TqaLlk8+VWdZuyQ@#M`%H?GY}Mv9K)iN#KJauAzKjZ&z%A;#F56`4G; zfN)TO>?#E`84;oEbO~}Mp$&B5%WI9tL1JQmC2%Lze9PVDu~o>Odt?Oor1|lUvrULX zUAd5ZvfJ6sAFdIioZVNPDT90+2xP_w8JJBtvWXZ!$TWk!4a&zs!R`D!2SfhXX0?JC z1_cU1Vih3@tr7x7I;T-q+j5*UZrOgi#=fj90k5ef%qfPvO#X|UxE{5U4CAV!XE*+b z8g^eT{C5N+7XJKV^o3}Ss^i?a!jFxib}!}nFh63YWf^OGI*;n7=I)&A zV0wuoH0dR5(cvN~+msvufB(zTn@e|$8zd-WNLPeL&BI&(Qx+i-aHFVPA1fqCw65hV zt{y5FKYX~pD_})@GR~pi1Or~miSu0#1Ku=}I<)>>LI^?CnsO)893dX!@7x){=R%Vz zOTylIs!89q^|NQ@1Hl`f7b`g&ki_PVRN&&bcRSM0$TO5LMTmSonI6PY%C)XRGQ8&2 zn4ALz4J75iEc9KxAL-7ub0ypK{{lskgx)Q=Axg`aq;;Lte0FjbMGcX# zSd1aES0hhs!wJ8eqKO@#f$aI|4I0EGk%w`SGrcx@upr5Ks4o@7v2(MeklYor_?#VJ zO--uI2^P$i<`D?K#?)v}`n@gNa6IKVVHuK08W>v$(Qf{6;j)DDK6>$nTpkjZ2vzww zQp5b9J?+Fj6Ph?UdTG2sU5p_N=p47_x=gK@329@ML}`Pg9=y~$Ep8HJ)uM|**z6ab z=OiestTi4zLv+8Bq6PUIw^0cNOL3h_)sLcRlyHfKXW!`QKmQ4pbgIoSZuT z=!S|e1jrzlc?Tha5uSSEBX|Qe=QjYu%6_`R6$G52GwQF4Hm{#Em>-@MoL_z*&MxOG zH2ZFIm+)9V84eoqsf9eMkahky@1!RKFf@^3O59LNxTsSMaWG{;zd!sQ>>GJ#>MHk} zLSZWF8#1@hBwBliyW5cV8L`E&pHe%HExf&iU(#{#`UkYHv=Jp^_4 zY>(oL{*|cSF-0Guj-~UJ|MkHoFP_h|d1xlo)5=>>B&dBS;3!YdXsc-^fOeTYx$Own z!(-;)dAjKjhO`!*JR<_JZl8mHC6|ebH6o6rny=i{!?u#h*=@g)hU zj=R(0swOW+H@dvEy&oZ}(qkPBJp#pJUm}p?ky)SeC_Ekjs&L_*aIT7K=376;QMLv0 zrOCE(U2XBZ9NP(n6(9vKW{@Pa8ud(mSn2SjFXidWXhWU`2|a)vMB5Po$Jfh}|KJv@ zL&OYl(Z6J1*z&S@{A!1jg2jyR`oW8z8gwUefh1dM0L)*V=ygaMd3M(^_fCJz7LNNQ zZXRF`d1%MBdtI&AdN~%se$LDeAyj~Ny{xj3ve;sVxb7lvVQqK9s6ymo@tj7do-q{Z z;-Me1qUNisl+LF0gHbP&uPS~-$)+>tN6Qs8N|GGO_aF5~d+x3-zIB3U)IwbxE_%ZZ zdK}G4>2cUCH5V#;nSmkvmMQjy{E+=DRR&Dx6KhFs_^ojkxk08Q2-v%4t`4S25K({& zp2*T0E7WQ?F=E%X(d!&X&iaQ~jhG#0B5->IPx4)7tswMX200O# zn;#G}WaB~u;Y4OHPj~E@|2aWWld*UT@k5}@VS?XFzaNw0R_<(RuO9Z#aR$~cQR4|v z-6C#4irQ!(c07_YiGM$Rymzcd_X0DqMQ7$WVt#7oiqR(p-*|ze6UdB24rV2&*$LrW3atF1-sB;Nfz$xu!9Aea-dd)h8>k9`WD?5BX^mGbBY1eZh9Sa`m+% z`Wfb>5FnFC{>N2uGGZ0H{2AT1Jh|~uf53An;hO9xxf8BSw<9sVEwXLSVi=bV1ZA{k z5_ZEW5@6oWFPXp$7DnK2p*tzHPjpF#$_#&N{zqUP1Yv?Y1rG>FConMEW+I^fqA`c? zW1fTCURWAE*vPL{T7iC?|B|$eQODN)lH-__oTEj`QvX&X$oYHhSDbBkxS~If_;gH@ z&Rp9JXx3oh%1rz$=ZW$nB9nl$C~$yX1TyRl4{^K>iyD{PO1~q zZ}ww8BA`oWhSo#GceWg2lT+J-5QKx95|G{W8xoVTE`i$8jZ!xJSiLJIz2mq(eVe)* zm`yw!l)ruVo2+?8ue-}Kd!zWuN@PF1v7^BX_71;FH`3N0A9QT=fPsIk^`VE&p15;1 z0Z*{fAvT5_kFO7lS09r&&98g%;5($&(`u3dn(ToR82Us68@Dw#d3oj)j5u(r)a>t| zg4;m>Yh@>WxqEnjLlPHhiaiAVA`R;JQ~%~Gk#;G6uRW*g+>)r4Lse^0&MX66)PG%I zA;kWTL^7f1T;dPPmtNJ&i2($$!6HmJE75=6P_70V%?FEDuz_Lgs*yuX*?oVn%fA9M zLDZ(Q&;ZI$&(t7vU?WbkyHmvhs8DjJhAC9Bf~49eC5NKNrZBV;JD?`*ItHDYdsf?uL71XP*F=70@?l+EJFywt5Y#7(;zJS7)8{T_ z-Cw-sUIp7k?jgN6t<8N<5jd*z8GsT6IjK+^`uP|2OHSuC38zcF&oYCZh-cxQ3*^CL z2T~(R!Lo0<5>5y_R%ma*fgJ=LHU*(dQ5i$i2}=Hiiy+J~SGXr?bc_CMuC5u}Wg(>| zq`&Y4;2W_1OS57^S%!XU%OWubdhqSvYYH$W3{o;fXD!N4P|+Lb%G- zu{$tjcRrw0o@imM=}79abhQ!azwpe_>}e_~-N1|Kgb8Q>)#{`d-#0DGBjtNz;79+G zTpwjbhAcJzknEA)0}y2jM7!(0euUcGPR_4iTJWr`_KOfRY`NW`oQNe~P+9um7exoH zuN@-=NAgp@_{Nn(SpsPSrvv0awjG4I_>n$nSmQz>YhhqRN3lD7LLqWH|EHH_Zm=W5 zY&L+-VA=SOF84Su(D?QeY6+E_=cu+zU2L1Ju7>g(Gkxy3k-Dx}zV*kw!bVG#5-RbH-;M$dA!!AmfV9HuxHU=Bs28KQmG)hO{?%AZZlv^+#3~aAX5ZE*1;g-#sK^D;niE=dmeI7t`8p4cuD>RZWPG;vCj_m(OF=Pj^q>u&*SP@qN}}2g?fyk7NPTr@OI&D9q%o+tQrX z!^~w;(j{e%e-2w0#W3W*wVpsYUMo;IpD=+zr!igV=X0}xUlC)&!*eK(tdHPNbB#vuChQ<)VJip{Ku5h4kmjegX3@}iedRABSEwID5-!dLch z9GLl#)@hSJaRLT2;fO(wF%j=5%DZfzoiBmN3>0<0;LC*beAj&amlOx<@hXooxwq>6 z&~}<%6DQU6^QU2MHPOc*7jOD;Bey#2v$8h-nOnpzb!v%`ZJ+=Z0FDn72(95je6jJz zgnCMNvdz~vB-;mO_)soaXX?s>X~x96a?y^|U>H!%{Nx*72ETiLt^HtBA@XnWi^@Xk z`c}dpIgGgp!JKh`hj9I(}yRRzh= zecEUjNKW%zQj0Rrk%!j4zn*yABL2@-BOL0|bK+tdR3HFE?_L88HaFYfS2K6M?Lz}KrJ2{U6V|JBZj6U{Vpg6?%pA?_9q45 zHm%MY?DF4nZDjMD`ruX&G1&qkcsbmF=0x@lsdc z$f+_boZ{pI)lEol=e3P|^v}txG;!3>vy(t2ExcZi;SzKa$3--gqlTcBe~tCa2t>M+ z5}+-yLfzO`E3`raDy|peaS?gujs#$1Jg8!Vv(I@^qt$pZT)3Ob_0641h?m$1${UK8 z%}LkWpvsJ&>h9q4f8#$aeQ({u^@f&>g4GZ2Pali=wLxmqtg8c0zq_(Nc9o`6nI}nA z4-^NBznZtth=8we-q=T_5|N_oNOEp^y{KOw=zy7Vjyiy*0)D=YFonb8R}o@SAhcXF zzgh{wuab`%d$;)G4|Q&#l9m^JE+$+FlGEXS;cJ>(I`O=HI)KBgE}@+06Ww-h0uR1l z@ngwJpq^|N#ltOnfQ-mieP3(DX-_-y^TiAt^-`m1q2##7GrOy|A^YDL#3EzwYlRbP z>Z}idxxHmV&_ZIgxm!Gn8m^z{yU)CHJP=_@`617SDNxL zYBUKpdt)$#_O;8wYulGz=`R@bsT^rRUT+opb`x8_-4$qW&D^4qjmIVvU^r#J4|qfI zKq+*CG4(sk@u=q(4|BIy$gZ?ttZLa(rcQeRN{mo@e>U=O#AgulSth4~K7bsW@Q3qm zJdNQ3e+F5s3oDp@zF-L2Ds5I+&v;f{B=gyEZ0fi@QW~uaz6Oz;Jctda$S^UNcWCg} z%?~0zJ)?7T(c2r&V-WlpNtZn@SUAka);$7NuNomfYr81`vKBv_=d%EsrLykN6X#LovS+p` zM5t$cAodF262>7ittV0n@AsD-y~U5QeXmr-gJwXEbt4Ex%M>&ge*at1>Lr#K&}kT! z*aU1Ea0f?|%~yV$P-3#L`UC6wHI?4&6T4ay=T{E~C{X=oHyXF8 z#?~3I*5X@W{^?~;u4I9j&PRw3k_EXisN6oH5AX8HQ(=vAj;HcD=E~Ob8r=l%EP<$b zgt6y}su9d)B;g`!B{)w(`D5Pc4K$hos6fCHITeLFWZ5KgzBaaZ_q_{f|C$>#p24K` z64#-FmoE_W>JVrT7fp^QQ2SH%LvCKy=+vs`h90?g_cxonpkjmvb#LdQnS6Y7L(}N@ zh$B~f#HIA7%y@$^elS0Hdfh4y6TJOMGd#<0y;S*u5705Ty@^%>uj9O(R<{WVLnulf z)hEEK01UKC(r6@2L%e#~`V0j-z8%_jy#`p9+CWk2z{_y$Gw%e2fz-FPaJn;w{T$J&20 z>R1L_eQZySr!BFdj}-N%9ENf`&+WK! z>cntHChd|?RAx?SEgwa&%s;mas<0T+N1?SVQu4bhKu&eXbH>@-KMMgbUuP*5Y=ZLe zr$1*JCKkJmT;1TG`Bzj9r$sGbvgd4>$h0UFkigqCSf4J&%(85c+@FEfYIvahw1}<-Oso% z@pbW3Leo*zd&TrnR~RZtI7df>z|B?w!v6arGM+ImI{U@_msD67pi75v>4^}5)3CG8 zT^~N#8=fZ)aZ|1gTKRPiW#T>9I4^>U^DT~(%=*cV zO(pLL|4_b2V)PsF6%vfye6T?BOr z3{?QBq5np%JEZgdGQ8irs+k_T6n8l}))C4w*TZK3?&V_$3xMco;bzq-=byVUlrX&a zh_?)3ql%^wJVkfhRy@tV_xHUg0G~ptz_S~y?!_N9kFWLj`NPhr>PGcuRc}n)hUNg@ z99MA4NA(rKEYMhw_O;)N^wuUmb;#3g6r>wpRLUSr2PF~^Bp)A7;XWvU=)V6^TSS#h zewmd#HysO4^|sJ2TfCAD!#c9ql`U;_bT`Rd@PK-~bxd7hxHfyvIj2kvM@;5(UY0U+ zjqX8ACHNit{CHzZeE=MfTa5dQosD^ZK zkELb8O0cQ-EX0+4(E)phwxrhrJp27$Y7s~_&}mhIng=kXzPwU;uuPX-C5&JqSzCKs zbiVIO2K;dta#{7%$-KWEYtJGHzV;F$vs2L|n0kiC?AoOyU54?E-y`w!^i3U}aUJj) z0vgTn)Fj6Ja3ueeo5Qg9_<4Epz3-MP^G+*ThLkFR1EN*>&w2{jcmdL6mr8Wf&qLMe z)r-l>vh%~CcdcLd#IRHaF^hGx1st5&&Q0{4xqSl$Q zBB_HP<4+>!iaz^@Fg+_4v`m<(rPH|J9|?*f1MC|1Zp^Y<0XkR2zU%;5c?A;Fn-ovG zIIecjWF7D99G1QU;c$+~y{}w=D~}%Xd}K76-x#2$p=5w+?Bi8~(IWO$x7fLf^&Pp^ z;E=t*N+BlKQ#~hdY@ucF(MkD|sF5F}y&F8jypO1>Q%ny8+jF(pXiy-Y%+bF8V) zqs`6RzM)>Rrp2*Akvj@>Dx1HpQ1OLF2iElG$_J)9%6v1F!0Meh-cBoz?RBNV&AbNV zPm4Ws+vuz-wnWOZu!$68U+P}6wMarWAT|sI(8KHWCP)^`H{V1LYlVe&BB5F}h}!#{ zkgDUW8RvE>T`tY7E1#j~y}pPerNz70zJs7sY|v~)#!XYm-W_yVk-#gd5z{yL+@udY zU(`J6*-v5yGE87z+H6+9>hF=HJh@t6jPWxfG=oCv?#NxxIQ{!Y-W0 zL$$tLFNcKR9>Szgj{eoXVG6jUMpm=c6Z;yC-RbhKwvE<;#PK=;dZ*L~XmGl1YFkw_ zGK1Ar0MM&HPC_??4V_0YrtKwIX3_Kde=x2m6;MZ2^vi&+nY)%1!^lRHSb56j;Br>s zGKdBwAis8{Spp2Z02u&1tfk4BdbFr`xPPg|o)BrwbCZ-B)x=Ua>RX+)@`0gp zGSAkJ6qpZW^&N)@0s)HD9S+Pu|E_WDM zsAC0Q@9GrvOnuG#cgcWH*N=yCW`FOM2eG~QCH~9q4;c(4$s!szDEUQtLiWCI{c|YZ zQA3BjAy^D-TnSp~1R7nUHmyH!U{p`eBPZXQ%o*+US3kou+*dEzuDFjq)7efUeR9pt z=bgd5kgVn+=hCZdmiy|SMBu8we(DglA)IU)5M#WrTbexthV;{6;EU6B58AThD&fFW z9dmd~mXcE$y@~WkN+-0`CU3NJN)|Rfw9tkZB-^@oMLBBhbC>y9uU|s7BtegcoX=-?|)Bk zd^%YCV*HS);lHkbo!Ty*+ZUDqHkZ@y>1Df`A5k|D%?}dQ-@d zq3^UC7*ZfP|6+6S(xm}UkbsBbouxr^z~inDAa+&RN1Fw19}`gn;{t-e_=hrXo#I&5 zh^JK+7v(BaK+baKpRI;C-ItOsqxqdY=4Y8S@?;IHPNN@eBYjN+V;<^uh0f!;*{WDo zK=n{PbmSI5i(NL1?+F3l7`zqsF8ET$w$OhQJduF^)-b#L;Kvh3c^lj@2yIqR;$HAN zdeaL(Z^59sD+;Djm5Q2A({m#3ZrFB9+seAz$@*1>@BZ_r(T7Z*!T=Q_I!jaW2_J4? zHB1GD+-9179H-z$4!*dH0cl>j-DqX4<^ak2Be_ zIJ{^iwc#Lip4S;$l*#}5XngK)Y2+?NDtqXftZ_v0AGy_D@e3Y;&My@<-d=Z>L6SH^Mzqc z0W^{Q4&g>QepnW*gdhpHZoYt^D$h+(dxU|b19JqxC8#?5bRhfyP5NlECChOpcmqwD z%7FFG4R)f3Z!a%AlZhFhrcH4>i&HNStv3R#zhlxL(TNl%kFW)Z$`97SDD?uK$NN*X|0ucR7S!q*Axrb&f# z7T(NJjbQ3kAQz|dBhR&cpx=*xQN{fQnhMBbfd*vHU%w93{73EEO4rpKRtDvY-< z4;6Mw>vev|^$Sb+A0IsC6!8Rol(QW|<^u#=t=~1Cy~d65c`-(b?ELc9$rRn=_3{42 zL)Ef}P8Hu%9s9?h+sH6GieLKE>IJ4}7+_W>n#5J#c`c}EnDXN_?~m#OF= z2{}IksZ!4Cx98$3uex%m(a#|x;qN{f>1L$44$ZyFF0Wr7P*vw@sjObjNHD>17!U#U zS%-zxDX?-2;*}y-6yEVf zHx*~;UZKT)W7AIP`=_=)9~%D+!V~dIG`bm6kQJxhTtfk<(Tt7;Ln;($p=Rn~;S>30 zgK6fnEZaI!brQ?-VV%`k1b|H`t7e#_zpdHBW7+gQ3zTL)x|4aTPZ93y zz@s+7sdRl5MbVKPGAQAk($9ENMHp|UCUVd1Q549WshF?LNB+Bt>FnUqJ#a5G3}uo3 z6EM<_2%@=i&-;FMO&-c+9V)SyDu_wwgbsaZPkb|M)2o=HTLZNrGH|}Nr?PL?N?o!` zCK;4Y6jlJ?(wqUnWjNTYKmtuVy{FLUdv;@PZ4meOlC3J80`#ptDINt;uEPTR$;3$i zgkR4s6u{z(-xC|?XZQF82ttK-FwtCO$5KuLkWWnHMGpw|pLluxkQw=7Ekw3~bV3Rc zf1y}P7&SoU_NdxQWZV8Cw|p zk|o)xgp91=W64@shLk0HWD8^Lp%ht?y;LgNJO~-emKZ{wBu4hfW2X^geBaMM@jZXR z{lhu;Irq7)*CpF~38~$ZI)=BsNc|R>r$CzQvexn7aMtPm@V2SZ`JIKqe%Vr@y$I=j zW!l8gbFDTglGkOp|7~9_FRN$jS93oaku9$A5;d6HJ~7PRE+3`i4ANgDDG*s6vE1*W zyPMrcSnD>J*yzxQtIMY^-`@B>iPB`$4SY1?)z4lp`vwBCz%M|8v3L$A;N3~st6((k zVX8q+7#nFP)rL(7*rNw!tv=Tr#Za~_KvpZ!=&SX8h&f*7kdonGJ+pH z;bHz@1JbDNox{QAhfMkfCT1<$kBv$*(0O`>u^K-G+^J{iLy&`ewK&G^oPT?8O8q zRu|5MY~(WPMxOXNUra}Yv6JdAZdo;jDp%nC9)c_Sek$IVvSgZzkr9jvRvex+lFjFUi0-}A&u+9dL97ZD)3x{Plhf{@dIL(#>ac1KD0fbnE ziSr-uuct}K9^Y|kH!3jib@cHh^vQn^PgGjBmngjP_GT|ph0E);dK)2us%lI<>1MzGKCoP${bLaQ-D~$a z!^eLmx~lt+$1$G_yl#i3&K=<7dD_G+j3tQ2Jh|-7Z=B?Yzw3|Zl-h* zRP7=b&>&_biyY1D4`s2ZU>J{?`wb^j$e7Cp<%vZ z7QilS~jVu=KVem!*NZifRIi^*h8FaNO$H+Hy z_?+7OM;z;fG)Pw3mMZ2_LXNf-GX6}wAa^tiArxmK1=63)5E-kJ#=rT|H3z`F{xDw! z1_CsJnp=VI!o{L$M4BW=qX{U#{8y6YtC5zo4lJ4L*G4tB<%A=@7bWIxH1FSvJsW2- z`opXo*tX~Zpv`DJgJ7h6*)6Iy;5aoJY6Z|0~dzV}vw)tK``jeORsv>|$)eKLaZ z{yywB8Il@&Wo4VhX+F}5qpG!zp(5Vqc<%0H5d0aPL`DknQC#>8eHMV)k*0*{eK@bf zt>KIQQoDR6$`rd%zhm|aZ$Rm5;01W(k!f!)c4NfWt6#;+|7w;^p|#oZuX1=XFCxR&8dhCV*zhBJ^H6rHP@)&_7d(~NXMjR zQ||Jy*%@S3nGHN6B_BZ`VT0&Eci5J+9oLtx(aEBL`%bn$GyD=GMs8?R@7&Pso$@bv zT+>|G!w=l?&@39LR4t*z_9N~pDP0Cce-HkULe)BhtkF;L{*0Ee=MavJmej%$)C4W- zw^+5+v6YEM9X4^AC3n0b!zVuicAqT}>=jSmxf2t0FH71kq5#)xk1(~a6Gc#9`DyYH z&3RDtZz3gk*KbFt0^|e<0t~tbju`NJfQ{b9z--B_Gjqaqcf|Ul1{!#gB^rlUxu`Zb z_%;_}q-)}lsVlPg#yo`*xFO>AEg0k9y3_lY1L@drS*GeAw08WN=w^U>U^zk&t=#kR zRD?NlR7ycgxH>=JG|gcQIF*TjLx}jruw1I;ro`e|!vhMj9n+#KkRx z#LvcyWAwT~d5S^ZU4-keo>TBi3dlxy{SZ5lLZH8`p=1&ey$zhe5D-F65QuC}h~ZF; z(?+(%J`lPZH+oX|aNHonW38kR9~F8O7sSo+pr8R9J-VkK>9>T5G?;!O&Lh9Pmr)}K zXofDEUTOj9Acz(CPTNS(??((s@(uU*1zZywpK&?g;9&z*Hs1}?g)I>54VttYZ21TF zulbQc>wkiy@BPUC6sI3dvUkv1a(hxaWinCjoqy{JkM703adX~pwLGv8zrG_#ZCvZF zV_oAL8Q+&wR%A8ee>L0gGL&_A1Xf121WukkbRK@9cwv$1ogj<*Ze-n;hGr%4xAp_H zEwGZ;;u@3&2|D?RIT!8%PX8HxyO$-H$=y)bWf7an_3UnsjV$SAhK!}pNS2cDx>xiC z`f}V#FyDCx{X&Tq@#Jq&Y&i>h*qoED2PxI>QO4Ti5@!$MjYA8_)0w!`O!5(E;usjr znW}5pyX671@mHaHab87XPFOH=SQ=vv(CfXr>|orHTe2`+U`qD*AMx%mU&QPTpGkc@ z`RR#v0a4Db6StD}s*>rgGAYgzGJ~pmcOcQ!L{DsI*-))n@AcKRp$0uHXWOQp)ntf) zvHt{1O4s}L%JFra&_NRJ3I?#WV6#48`oz;g%K?pyra2)<^9SL5!9u{cKjV3}LmJl} z@pJYUZTfZiU?Gt4PsEYvf3+k-(tsoypsm=@R=fa7d^(fyO9bWD5d}I<9skpf_D4$M zB$R40)Nw~CjwkW8b|Klx*O0Csr|}$guEerv2L~A}r$0?f?JG*EE&zxj5q4=Pj8;Tzc*Y{}$tyt3Qvq)O&OCWhJuC8r2 z_*ekCn8$VUBk3$2)sgRXj+pQD(T6+iI99%Vxs|Oxj|`c z_{^9R^OwonPQkwYq97$?M==M=?5~-_+~1`3;2GG!XvYRFL9_}Ug8aQKp@Rd;KK61K zHN2ZZ`#N5#@J67mVwGp4$B9HSePk>Q0$2$B`a}lZH|-`l|F=7QHLQxFjgRg9<_Dc`nil27K$cPWF!2!(SH?h0sxyY^xLW!1Q{Ywe%o zOs2ldRE5c<{|fNFK%Lun&JCU3SzP;i1P-(@v1sJ&#QXXb#{?U@)`w}< z-kG(9g=az(4T;(pcxpdsf0i@Mejyir>ggvy6f!8`%CYgb5x={RTAXatQ?Fg4cIeVhOZW|sTxwP;Aot%HU>7xmVOE+#!U=hRSt48_1ud+C@sTq;( zOFGu_unw&(AFTC~cVth1jCW0)qw#|FpWGyA>7gdD(Q$qh1)eTtx5Z~Hr@8b{N2V(i zufxnU*YVCvl_bW5z zW7}&OR$}K$Uf9R`cZKRRGE?dkkM7#MeoP*3adnW%q?9kPpyxYwAXZXZn&_0_y;a`I zPhyXcwN7;Yx>h^M*mUsD3o`8bG_pa+QHL6HgP&cFAxbfMT)Fx!jq5}2mN&4LB|WAD zG+yPfueFqS%2E2q96wsG?{aU-7bJNEXOmoNsh{%1kPj1XS0xZtPEVJm z9iG%Ed#cQ7Qt$Zzwo+&Uaa&rbm2?yj$s+9@Ux!@HeZzH=%iLAu zwqClI3ZLP#7m@67WmZNYrZApa2}kyF67tst@sJjkRp<-P$gUNHkx*hP(gOP+wveGx z(u0i!90(fc{rAlA*w2*3eJK4it!bNYH2hM?W({YBeE}{;!SWzUc|~2_%`&`Ga@hXs zhse?<-$%iCoyFqY2`go$t{cMSaScAbRju_FdnvavH zuv>HcFPQTlhG=og3X^HmcueXUw9;gv%eNQ=_os7IU~UKyLr~>|MXYIpLDZhc!}atI zC2?};OP*wPVQMp<=1sZk8^)|7>u*!~1^fD9O;M0U+7T$}h%cq2NuEwqEPIIGtsuYHFW6ayFJ5qvtGjZyp>)@*SRKjIN7O0PDrM$)-$}s-Q?-Q zW2u4t2^kCd-L79LtLne5QqAA|)a5AK$s$l=QY&(6E245^(c>32F^=yg3^kui@S8bi zb{2keM{ORy}AZI{r{~z$3uyyLwNyJKl>wK-5Q)Tu{N$fOC=2.0.9 pydruid==0.4 requests_aws_sign==0.1.4 snowflake_connector_python==1.6.10 +phoenixdb==0.7 # certifi is needed to support MongoDB and SSL: certifi From d5494cff086a09017c401cf381bc9b1ab46fbd3b Mon Sep 17 00:00:00 2001 From: Allen Short Date: Wed, 27 Mar 2019 10:50:39 -0500 Subject: [PATCH 009/179] Fail query task properly even if error message is empty (#3499) --- redash/tasks/queries.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/redash/tasks/queries.py b/redash/tasks/queries.py index a83ab3242e..fa4b9d6891 100644 --- a/redash/tasks/queries.py +++ b/redash/tasks/queries.py @@ -323,7 +323,7 @@ def run(self): _unlock(self.query_hash, self.data_source.id) - if error: + if error is not None and data is None: result = QueryExecutionError(error) if self.scheduled_query is not None: self.scheduled_query = models.db.session.merge(self.scheduled_query, load=False) From 49ffaae3ecfaa2ef234d321ba870eb37e54eac93 Mon Sep 17 00:00:00 2001 From: Omer Lachish Date: Wed, 27 Mar 2019 17:57:51 +0200 Subject: [PATCH 010/179] Fix email shows as unverified when no email server is configured (#3613) * check that e-mail server is configured before marking the email address as not verified and sending out a verification e-mail * use helper method in `invite_user` * move email_server_configured helper to settings * add test to verify that email addresses arent marked as unverified if there's no e-mail server to verify them * simplify a couple of tests with patch * combine conditions into single variable * Booleans, gotta love 'em --- redash/handlers/authentication.py | 2 +- redash/handlers/users.py | 10 +++++----- redash/settings/__init__.py | 5 +++++ tests/handlers/test_users.py | 26 +++++++++++++------------- 4 files changed, 24 insertions(+), 19 deletions(-) diff --git a/redash/handlers/authentication.py b/redash/handlers/authentication.py index 610575c1ab..2776816033 100644 --- a/redash/handlers/authentication.py +++ b/redash/handlers/authentication.py @@ -224,7 +224,7 @@ def client_config(): 'showPermissionsControl': current_org.get_setting("feature_show_permissions_control"), 'allowCustomJSVisualizations': settings.FEATURE_ALLOW_CUSTOM_JS_VISUALIZATIONS, 'autoPublishNamedQueries': settings.FEATURE_AUTO_PUBLISH_NAMED_QUERIES, - 'mailSettingsMissing': settings.MAIL_DEFAULT_SENDER is None, + 'mailSettingsMissing': not settings.email_server_is_configured(), 'dashboardRefreshIntervals': settings.DASHBOARD_REFRESH_INTERVALS, 'queryRefreshIntervals': settings.QUERY_REFRESH_INTERVALS, 'googleLoginEnabled': settings.GOOGLE_OAUTH_ENABLED, diff --git a/redash/handlers/users.py b/redash/handlers/users.py index cabbe39dc5..7a57e0a99f 100644 --- a/redash/handlers/users.py +++ b/redash/handlers/users.py @@ -38,11 +38,10 @@ def invite_user(org, inviter, user, send_email=True): - email_configured = settings.MAIL_DEFAULT_SENDER is not None d = user.to_dict() invite_url = invite_link_for_user(user) - if email_configured and send_email: + if settings.email_server_is_configured() and send_email: send_invite_email(inviter, user, invite_url, org) else: d['invite_link'] = invite_url @@ -229,15 +228,16 @@ def post(self, user_id): if domain.lower() in blacklist or domain.lower() == 'qq.com': abort(400, message='Bad email address.') - email_changed = 'email' in params and params['email'] != user.email - if email_changed: + email_address_changed = 'email' in params and params['email'] != user.email + needs_to_verify_email = email_address_changed and settings.email_server_is_configured() + if needs_to_verify_email: user.is_email_verified = False try: self.update_model(user, params) models.db.session.commit() - if email_changed: + if needs_to_verify_email: send_verify_email(user, self.current_org) # The user has updated their email or password. This should invalidate all _other_ sessions, diff --git a/redash/settings/__init__.py b/redash/settings/__init__.py index ef6ad136a7..6f8766a017 100644 --- a/redash/settings/__init__.py +++ b/redash/settings/__init__.py @@ -215,6 +215,11 @@ def all_settings(): MAIL_MAX_EMAILS = os.environ.get('REDASH_MAIL_MAX_EMAILS', None) MAIL_ASCII_ATTACHMENTS = parse_boolean(os.environ.get('REDASH_MAIL_ASCII_ATTACHMENTS', 'false')) + +def email_server_is_configured(): + return MAIL_DEFAULT_SENDER is not None + + HOST = os.environ.get('REDASH_HOST', '') ALERTS_DEFAULT_MAIL_SUBJECT_TEMPLATE = os.environ.get('REDASH_ALERTS_DEFAULT_MAIL_SUBJECT_TEMPLATE', "({state}) {alert_name}") diff --git a/tests/handlers/test_users.py b/tests/handlers/test_users.py index 1541e12e2b..7f4d55e0d8 100644 --- a/tests/handlers/test_users.py +++ b/tests/handlers/test_users.py @@ -40,10 +40,8 @@ def test_creates_user(self): self.assertEqual(rv.json['name'], test_user['name']) self.assertEqual(rv.json['email'], test_user['email']) - def test_shows_invite_link_when_email_is_not_configured(self): - previous = settings.MAIL_DEFAULT_SENDER - settings.MAIL_DEFAULT_SENDER = None - + @patch('redash.settings.email_server_is_configured', return_value=False) + def test_shows_invite_link_when_email_is_not_configured(self, _): admin = self.factory.create_admin() test_user = {'name': 'User', 'email': 'user@example.com'} @@ -52,12 +50,8 @@ def test_shows_invite_link_when_email_is_not_configured(self): self.assertEqual(rv.status_code, 200) self.assertTrue('invite_link' in rv.json) - settings.MAIL_DEFAULT_SENDER = previous - - def test_does_not_show_invite_link_when_email_is_configured(self): - previous = settings.MAIL_DEFAULT_SENDER - settings.MAIL_DEFAULT_SENDER = "john@doe.com" - + @patch('redash.settings.email_server_is_configured', return_value=True) + def test_does_not_show_invite_link_when_email_is_configured(self, _): admin = self.factory.create_admin() test_user = {'name': 'User', 'email': 'user@example.com'} @@ -66,8 +60,6 @@ def test_does_not_show_invite_link_when_email_is_configured(self): self.assertEqual(rv.status_code, 200) self.assertFalse('invite_link' in rv.json) - settings.MAIL_DEFAULT_SENDER = previous - def test_creates_user_case_insensitive_email(self): admin = self.factory.create_admin() @@ -230,12 +222,20 @@ def test_returns_200_for_non_admin_changing_his_own(self): rv = self.make_request('post', "/api/users/{}".format(self.factory.user.id), data={"name": "New Name"}) self.assertEqual(rv.status_code, 200) - def test_marks_email_as_not_verified_when_changed(self): + @patch('redash.settings.email_server_is_configured', return_value=True) + def test_marks_email_as_not_verified_when_changed(self, _): user = self.factory.user user.is_email_verified = True rv = self.make_request('post', "/api/users/{}".format(user.id), data={"email": "donald@trump.biz"}) self.assertFalse(user.is_email_verified) + @patch('redash.settings.email_server_is_configured', return_value=False) + def test_doesnt_mark_email_as_not_verified_when_changed_and_email_server_is_not_configured(self, _): + user = self.factory.user + user.is_email_verified = True + rv = self.make_request('post', "/api/users/{}".format(user.id), data={"email": "donald@trump.biz"}) + self.assertTrue(user.is_email_verified) + def test_returns_200_for_admin_changing_other_user(self): admin = self.factory.create_admin() From 7a7fdf9c9922c6355aea0e466d956b0614a25b0b Mon Sep 17 00:00:00 2001 From: ialeinikov <30274504+ialeinikov@users.noreply.github.com> Date: Thu, 28 Mar 2019 02:58:48 +1100 Subject: [PATCH 011/179] allowing to specify a custom work group for AWS Athena queries (#3592) * allowing to specify a custom work group for AWS Athena queries * Fixing title + adding correct position in the UI --- redash/query_runner/athena.py | 8 +++++++- requirements_all_ds.txt | 6 +++--- 2 files changed, 10 insertions(+), 4 deletions(-) diff --git a/redash/query_runner/athena.py b/redash/query_runner/athena.py index 8d4f3dfcff..d9d2736531 100644 --- a/redash/query_runner/athena.py +++ b/redash/query_runner/athena.py @@ -78,9 +78,14 @@ def configuration_schema(cls): 'type': 'boolean', 'title': 'Use Glue Data Catalog', }, + 'work_group': { + 'type': 'string', + 'title': 'Athena Work Group', + 'default': 'primary' + }, }, 'required': ['region', 's3_staging_dir'], - 'order': ['region', 'aws_access_key', 'aws_secret_key', 's3_staging_dir', 'schema'], + 'order': ['region', 'aws_access_key', 'aws_secret_key', 's3_staging_dir', 'schema', 'work_group'], 'secret': ['aws_secret_key'] } @@ -170,6 +175,7 @@ def run_query(self, query, user): schema_name=self.configuration.get('schema', 'default'), encryption_option=self.configuration.get('encryption_option', None), kms_key=self.configuration.get('kms_key', None), + work_group=self.configuration.get('work_group', 'primary'), formatter=SimpleFormatter()).cursor() try: diff --git a/requirements_all_ds.txt b/requirements_all_ds.txt index c9f44a6a70..f40320cc2a 100644 --- a/requirements_all_ds.txt +++ b/requirements_all_ds.txt @@ -11,8 +11,8 @@ td-client==0.8.0 pymssql==2.1.3 dql==0.5.24 dynamo3==0.4.7 -boto3==1.9.85 -botocore==1.12.85 +boto3==1.9.115 +botocore==1.12.115 sasl>=0.1.3 thrift>=0.8.0 thrift_sasl>=0.1.0 @@ -20,7 +20,7 @@ cassandra-driver==3.11.0 memsql==2.16.0 atsd_client==2.0.12 simple_salesforce==0.72.2 -PyAthena>=1.0.0 +PyAthena>=1.5.0 pymapd==0.7.1 qds-sdk>=1.9.6 ibm-db>=2.0.9 From 973ad565cd3dc95a88bc9791f55585357ff35939 Mon Sep 17 00:00:00 2001 From: Justin Clift Date: Thu, 28 Mar 2019 03:06:40 +1100 Subject: [PATCH 012/179] Update PostgreSQL version to always use latest in the 9.5 series (#3639) --- setup/docker-compose.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup/docker-compose.yml b/setup/docker-compose.yml index a70a1b9925..e6b64ac951 100644 --- a/setup/docker-compose.yml +++ b/setup/docker-compose.yml @@ -36,7 +36,7 @@ services: image: redis:3.0-alpine restart: always postgres: - image: postgres:9.5.6-alpine + image: postgres:9.5-alpine env_file: /opt/redash/env volumes: - /opt/redash/postgres-data:/var/lib/postgresql/data From 872d0ca5e63acfd5ba90e4cc203de81b376cb398 Mon Sep 17 00:00:00 2001 From: shinsuke-nara Date: Thu, 28 Mar 2019 01:08:38 +0900 Subject: [PATCH 013/179] Show accessible tables only in New Query view for PostgreSQL (#3599) * Show accessible tables only. * Get table information from information_schema.columns. * Union old query. --- redash/query_runner/pg.py | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/redash/query_runner/pg.py b/redash/query_runner/pg.py index 048f5880f0..5524375de6 100644 --- a/redash/query_runner/pg.py +++ b/redash/query_runner/pg.py @@ -152,7 +152,15 @@ def _get_tables(self, schema): ON a.attrelid = c.oid AND a.attnum > 0 AND NOT a.attisdropped - WHERE c.relkind IN ('r', 'v', 'm', 'f', 'p') + WHERE c.relkind IN ('m', 'f', 'p') + + UNION + + SELECT table_schema, + table_name, + column_name + FROM information_schema.columns + WHERE table_schema NOT IN ('pg_catalog', 'information_schema') """ self._get_definitions(schema, query) From 375e61f2637c2757c036182a500fab22a77d370e Mon Sep 17 00:00:00 2001 From: Gabriel Dutra Date: Wed, 27 Mar 2019 13:09:56 -0300 Subject: [PATCH 014/179] Add error message when destination name already exists (#3597) * Return 400 when destination name already exists * Remove whitespace * Unicode 1 Co-Authored-By: gabrieldutra * Unicode 2 Co-Authored-By: gabrieldutra --- redash/handlers/destinations.py | 15 +++++++++++++-- tests/handlers/test_destinations.py | 11 +++++++++++ 2 files changed, 24 insertions(+), 2 deletions(-) diff --git a/redash/handlers/destinations.py b/redash/handlers/destinations.py index 2e51dc65fa..2112e98c89 100644 --- a/redash/handlers/destinations.py +++ b/redash/handlers/destinations.py @@ -1,5 +1,6 @@ from flask import make_response, request from flask_restful import abort +from sqlalchemy.exc import IntegrityError from redash import models from redash.destinations import (destinations, @@ -45,6 +46,10 @@ def post(self, destination_id): models.db.session.commit() except ValidationError: abort(400) + except IntegrityError as e: + if 'name' in e.message: + abort(400, message=u"Alert Destination with the name {} already exists.".format(req['name'])) + abort(500) return destination.to_dict(all=True) @@ -102,6 +107,12 @@ def post(self): options=config, user=self.current_user) - models.db.session.add(destination) - models.db.session.commit() + try: + models.db.session.add(destination) + models.db.session.commit() + except IntegrityError as e: + if 'name' in e.message: + abort(400, message=u"Alert Destination with the name {} already exists.".format(req['name'])) + abort(500) + return destination.to_dict(all=True) diff --git a/tests/handlers/test_destinations.py b/tests/handlers/test_destinations.py index 2e1533b7cb..e3a31d0bfd 100644 --- a/tests/handlers/test_destinations.py +++ b/tests/handlers/test_destinations.py @@ -38,6 +38,17 @@ def test_post_requires_admin(self): rv = self.make_request('post', '/api/destinations', user=self.factory.user, data=data) self.assertEqual(rv.status_code, 403) + def test_returns_400_when_name_already_exists(self): + d1 = self.factory.create_destination() + data = { + 'options': {'addresses': 'test@example.com'}, + 'name': d1.name, + 'type': 'email' + } + + rv = self.make_request('post', '/api/destinations', user=self.factory.create_admin(), data=data) + self.assertEqual(rv.status_code, 400) + class TestDestinationResource(BaseTestCase): def test_get(self): From 1933dee8ca816153000ab3385a9c5d59dd1e7236 Mon Sep 17 00:00:00 2001 From: Jannis Leidel Date: Wed, 27 Mar 2019 20:08:20 +0100 Subject: [PATCH 015/179] Fix Celery worker --max-tasks-per-child for Celery 4.x. (#3625) * Fix Celery worker CLI parameter name that was changed in Celery 4.x. * Set Celery worker --max-memory-per-child to 1/4th of total system memory. * Review fixes. * Review fixes. --- bin/docker-entrypoint | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/bin/docker-entrypoint b/bin/docker-entrypoint index c7d0e473b6..62d80e9526 100755 --- a/bin/docker-entrypoint +++ b/bin/docker-entrypoint @@ -4,9 +4,10 @@ set -e worker() { WORKERS_COUNT=${WORKERS_COUNT:-2} QUEUES=${QUEUES:-queries,scheduled_queries,celery,schemas} + WORKER_EXTRA_OPTIONS=${WORKER_EXTRA_OPTIONS:-} echo "Starting $WORKERS_COUNT workers for queues: $QUEUES..." - exec /usr/local/bin/celery worker --app=redash.worker -c$WORKERS_COUNT -Q$QUEUES -linfo --maxtasksperchild=10 -Ofair + exec /usr/local/bin/celery worker --app=redash.worker -c$WORKERS_COUNT -Q$QUEUES -linfo --max-tasks-per-child=10 -Ofair $WORKER_EXTRA_OPTIONS } scheduler() { @@ -16,7 +17,7 @@ scheduler() { echo "Starting scheduler and $WORKERS_COUNT workers for queues: $QUEUES..." - exec /usr/local/bin/celery worker --app=redash.worker --beat -s$SCHEDULE_DB -c$WORKERS_COUNT -Q$QUEUES -linfo --maxtasksperchild=10 -Ofair + exec /usr/local/bin/celery worker --app=redash.worker --beat -s$SCHEDULE_DB -c$WORKERS_COUNT -Q$QUEUES -linfo --max-tasks-per-child=10 -Ofair } server() { From 2699d24441fd82c535306888de8e4e1ab6c3f309 Mon Sep 17 00:00:00 2001 From: Gabriel Dutra Date: Wed, 27 Mar 2019 16:29:48 -0300 Subject: [PATCH 016/179] Manage user groups in UserEdit (#3450) --- client/app/assets/less/ant.less | 1 + .../components/dynamic-form/DynamicForm.jsx | 24 +++++ client/app/components/proptypes.js | 22 ++++- client/app/components/users/UserEdit.jsx | 52 ++++++++++- client/app/components/users/UserShow.jsx | 88 +++++++++++++------ client/app/components/users/UserShow.test.js | 6 ++ .../users/__snapshots__/UserShow.test.js.snap | 6 ++ client/app/services/group.js | 2 +- client/app/services/user.js | 1 + .../integration/user/edit_profile_spec.js | 2 + redash/handlers/users.py | 17 +++- tests/handlers/test_users.py | 9 ++ 12 files changed, 194 insertions(+), 36 deletions(-) diff --git a/client/app/assets/less/ant.less b/client/app/assets/less/ant.less index 337bc1d744..3d67ce8ffc 100644 --- a/client/app/assets/less/ant.less +++ b/client/app/assets/less/ant.less @@ -20,6 +20,7 @@ @import '~antd/lib/tag/style/index'; @import '~antd/lib/grid/style/index'; @import '~antd/lib/switch/style/index'; +@import '~antd/lib/empty/style/index'; @import '~antd/lib/drawer/style/index'; @import '~antd/lib/divider/style/index'; @import '~antd/lib/dropdown/style/index'; diff --git a/client/app/components/dynamic-form/DynamicForm.jsx b/client/app/components/dynamic-form/DynamicForm.jsx index b4be5c407e..8bfe38bdad 100644 --- a/client/app/components/dynamic-form/DynamicForm.jsx +++ b/client/app/components/dynamic-form/DynamicForm.jsx @@ -7,6 +7,7 @@ import Checkbox from 'antd/lib/checkbox'; import Button from 'antd/lib/button'; import Upload from 'antd/lib/upload'; import Icon from 'antd/lib/icon'; +import Select from 'antd/lib/select'; import notification from '@/services/notification'; import { includes } from 'lodash'; import { react2angular } from 'react2angular'; @@ -132,6 +133,25 @@ export const DynamicForm = Form.create()(class DynamicForm extends React.Compone return getFieldDecorator(name, fileOptions)(upload); } + renderSelect(field, props) { + const { getFieldDecorator } = this.props.form; + const { name, options, mode, initialValue, readOnly, loading } = field; + const { Option } = Select; + + const decoratorOptions = { + rules: fieldRules(field), + initialValue, + }; + + return getFieldDecorator(name, decoratorOptions)( + , + ); + } + renderField(field, props) { const { getFieldDecorator } = this.props.form; const { name, type, initialValue } = field; @@ -147,6 +167,10 @@ export const DynamicForm = Form.create()(class DynamicForm extends React.Compone return getFieldDecorator(name, options)({fieldLabel}); } else if (type === 'file') { return this.renderUpload(field, props); + } else if (type === 'select') { + return this.renderSelect(field, props); + } else if (type === 'content') { + return field.content; } else if (type === 'number') { return getFieldDecorator(name, options)(); } diff --git a/client/app/components/proptypes.js b/client/app/components/proptypes.js index 7567727fac..43f368cadf 100644 --- a/client/app/components/proptypes.js +++ b/client/app/components/proptypes.js @@ -34,12 +34,30 @@ export const RefreshScheduleDefault = { export const Field = PropTypes.shape({ name: PropTypes.string.isRequired, title: PropTypes.string, - type: PropTypes.oneOf(['text', 'email', 'password', 'number', 'checkbox', 'file']).isRequired, - initialValue: PropTypes.oneOfType([PropTypes.string, PropTypes.number, PropTypes.bool]), + type: PropTypes.oneOf([ + 'text', + 'email', + 'password', + 'number', + 'checkbox', + 'file', + 'select', + 'content', + ]).isRequired, + initialValue: PropTypes.oneOfType([ + PropTypes.string, + PropTypes.number, + PropTypes.bool, + PropTypes.arrayOf(PropTypes.string), + PropTypes.arrayOf(PropTypes.number), + ]), + content: PropTypes.node, + mode: PropTypes.string, required: PropTypes.bool, readOnly: PropTypes.bool, minLength: PropTypes.number, placeholder: PropTypes.string, + loading: PropTypes.bool, props: PropTypes.object, // eslint-disable-line react/forbid-prop-types }); diff --git a/client/app/components/users/UserEdit.jsx b/client/app/components/users/UserEdit.jsx index 190cc15ab4..4b3bcd50c9 100644 --- a/client/app/components/users/UserEdit.jsx +++ b/client/app/components/users/UserEdit.jsx @@ -1,9 +1,12 @@ import React, { Fragment } from 'react'; +import { includes } from 'lodash'; import Alert from 'antd/lib/alert'; import Button from 'antd/lib/button'; import Form from 'antd/lib/form'; import Modal from 'antd/lib/modal'; +import Tag from 'antd/lib/tag'; import { User } from '@/services/user'; +import { Group } from '@/services/group'; import { currentUser } from '@/services/auth'; import { absoluteUrl } from '@/services/utils'; import { UserProfile } from '../proptypes'; @@ -20,6 +23,8 @@ export default class UserEdit extends React.Component { super(props); this.state = { user: this.props.user, + groups: [], + loadingGroups: true, regeneratingApiKey: false, sendingPasswordEmail: false, resendingInvitation: false, @@ -27,6 +32,15 @@ export default class UserEdit extends React.Component { }; } + componentDidMount() { + Group.query((groups) => { + this.setState({ + groups: groups.map(({ id, name }) => ({ value: id, title: name })), + loadingGroups: false, + }); + }); + } + changePassword = () => { ChangePasswordDialog.showModal({ user: this.props.user }); }; @@ -102,8 +116,9 @@ export default class UserEdit extends React.Component { }); }; - renderBasicInfoForm() { - const { user } = this.state; + renderUserInfoForm() { + const { user, groups, loadingGroups } = this.state; + const formFields = [ { name: 'name', @@ -117,7 +132,22 @@ export default class UserEdit extends React.Component { type: 'email', initialValue: user.email, }, - ].map(field => ({ ...field, readOnly: user.isDisabled, required: true })); + (!user.isDisabled && currentUser.id !== user.id) ? { + name: 'group_ids', + title: 'Groups', + type: 'select', + mode: 'multiple', + options: groups, + initialValue: groups.filter(group => includes(user.groupIds, group.value)).map(group => group.value), + loading: loadingGroups, + placeholder: loadingGroups ? 'Loading...' : '', + } : { + name: 'group_ids', + title: 'Groups', + type: 'content', + content: this.renderUserGroups(), + }, + ].map(field => ({ readOnly: user.isDisabled, required: true, ...field })); return ( + {groups.filter(group => includes(user.groupIds, group.value)).map((group => ( + + {group.title} + + )))} + + ); + } + renderApiKey() { const { user, regeneratingApiKey } = this.state; @@ -227,7 +271,7 @@ export default class UserEdit extends React.Component { />

{user.name}

- {this.renderBasicInfoForm()} + {this.renderUserInfoForm()} {!user.isDisabled && ( {this.renderApiKey()} diff --git a/client/app/components/users/UserShow.jsx b/client/app/components/users/UserShow.jsx index 87de7f5d14..8e41ff3770 100644 --- a/client/app/components/users/UserShow.jsx +++ b/client/app/components/users/UserShow.jsx @@ -1,30 +1,66 @@ import React from 'react'; +import { includes } from 'lodash'; +import Tag from 'antd/lib/tag'; +import { Group } from '@/services/group'; import { UserProfile } from '../proptypes'; -export default function UserShow({ user: { name, email, profileImageUrl } }) { - return ( -
- profile - -

{name}

- -
- -
-
Name:
-
{name}
-
Email:
-
{email}
-
-
- ); -} +export default class UserShow extends React.Component { + static propTypes = { + user: UserProfile.isRequired, + }; + + constructor(props) { + super(props); + this.state = { groups: [], loadingGroups: true }; + } + + componentDidMount() { + Group.query((groups) => { + this.setState({ groups, loadingGroups: false }); + }); + } + + renderUserGroups() { + const { groupIds } = this.props.user; + const { groups } = this.state; + + return ( +
+ {groups.filter(group => includes(groupIds, group.id)).map((group => ( + + {group.name} + + )))} +
+ ); + } -UserShow.propTypes = { - user: UserProfile.isRequired, -}; + render() { + const { name, email, profileImageUrl } = this.props.user; + const { loadingGroups } = this.state; + + return ( +
+ profile + +

{name}

+ +
+ +
+
Name:
+
{name}
+
Email:
+
{email}
+
Groups:
+
{loadingGroups ? 'Loading...' : this.renderUserGroups()}
+
+
+ ); + } +} diff --git a/client/app/components/users/UserShow.test.js b/client/app/components/users/UserShow.test.js index 7fd859e8a0..2badc694f3 100644 --- a/client/app/components/users/UserShow.test.js +++ b/client/app/components/users/UserShow.test.js @@ -1,12 +1,18 @@ import React from 'react'; import renderer from 'react-test-renderer'; +import { Group } from '@/services/group'; import UserShow from './UserShow'; +beforeEach(() => { + Group.query = jest.fn(dataCallback => dataCallback([])); +}); + test('renders correctly', () => { const user = { id: 2, name: 'John Doe', email: 'john@doe.com', + groupIds: [], profileImageUrl: 'http://www.images.com/llama.jpg', }; diff --git a/client/app/components/users/__snapshots__/UserShow.test.js.snap b/client/app/components/users/__snapshots__/UserShow.test.js.snap index 4455d79d84..20debdd5cf 100644 --- a/client/app/components/users/__snapshots__/UserShow.test.js.snap +++ b/client/app/components/users/__snapshots__/UserShow.test.js.snap @@ -31,6 +31,12 @@ exports[`renders correctly 1`] = `
john@doe.com
+
+ Groups: +
+
+
+
`; diff --git a/client/app/services/group.js b/client/app/services/group.js index e10aca9ef0..db12bc53cd 100644 --- a/client/app/services/group.js +++ b/client/app/services/group.js @@ -1,4 +1,4 @@ -export let Group = null; // eslint-disable-line import/no-mutable-exports +export let Group = {}; // eslint-disable-line import/no-mutable-exports function GroupService($resource) { const actions = { diff --git a/client/app/services/user.js b/client/app/services/user.js index 50b21899a5..2e3407dd5c 100644 --- a/client/app/services/user.js +++ b/client/app/services/user.js @@ -66,6 +66,7 @@ function convertUserInfo(user) { email: user.email, profileImageUrl: user.profile_image_url, apiKey: user.api_key, + groupIds: user.groups, isDisabled: user.is_disabled, isInvitationPending: user.is_invitation_pending, }; diff --git a/client/cypress/integration/user/edit_profile_spec.js b/client/cypress/integration/user/edit_profile_spec.js index 91e5236d60..f93ff27bf8 100644 --- a/client/cypress/integration/user/edit_profile_spec.js +++ b/client/cypress/integration/user/edit_profile_spec.js @@ -41,6 +41,8 @@ describe('Edit Profile', () => { $apiKey.val('secret'); }); + cy.getByTestId('Groups').should('contain', 'admin'); + cy.percySnapshot('User Profile'); }); diff --git a/redash/handlers/users.py b/redash/handlers/users.py index 7a57e0a99f..8549818828 100644 --- a/redash/handlers/users.py +++ b/redash/handlers/users.py @@ -4,6 +4,7 @@ from flask_restful import abort from flask_login import current_user, login_user from funcy import project +from sqlalchemy.orm.exc import NoResultFound from sqlalchemy.exc import IntegrityError from disposable_email_domains import blacklist from funcy import partial @@ -207,7 +208,7 @@ def post(self, user_id): req = request.get_json(True) - params = project(req, ('email', 'name', 'password', 'old_password', 'groups')) + params = project(req, ('email', 'name', 'password', 'old_password', 'group_ids')) if 'password' in params and 'old_password' not in params: abort(403, message="Must provide current password to update password.") @@ -219,8 +220,18 @@ def post(self, user_id): user.hash_password(params.pop('password')) params.pop('old_password') - if 'groups' in params and not self.current_user.has_permission('admin'): - abort(403, message="Must be admin to change groups membership.") + if 'group_ids' in params: + if not self.current_user.has_permission('admin'): + abort(403, message="Must be admin to change groups membership.") + + for group_id in params['group_ids']: + try: + models.Group.get_by_id_and_org(group_id, self.current_org) + except NoResultFound: + abort(400, message="Group id {} is invalid.".format(group_id)) + + if len(params['group_ids']) == 0: + params.pop('group_ids') if 'email' in params: _, domain = params['email'].split('@', 1) diff --git a/tests/handlers/test_users.py b/tests/handlers/test_users.py index 7f4d55e0d8..c5cb375220 100644 --- a/tests/handlers/test_users.py +++ b/tests/handlers/test_users.py @@ -307,6 +307,15 @@ def test_changing_email_does_not_end_current_session(self): # make sure the session's `user_id` has changed to reflect the new identity, thus not logging the user out self.assertNotEquals(previous, current) + def test_admin_can_change_user_groups(self): + admin_user = self.factory.create_admin() + other_user = self.factory.create_user(group_ids=[1]) + + rv = self.make_request('post', "/api/users/{}".format(other_user.id), data={"group_ids": [1, 2]}, user=admin_user) + + self.assertEqual(rv.status_code, 200) + self.assertEqual(models.User.query.get(other_user.id).group_ids, [1,2]) + def test_admin_can_delete_user(self): admin_user = self.factory.create_admin() other_user = self.factory.create_user(is_invitation_pending=True) From b3819de878f7bed936669e841cea99971fd24e13 Mon Sep 17 00:00:00 2001 From: Allen Short Date: Wed, 27 Mar 2019 15:00:09 -0500 Subject: [PATCH 017/179] Treat repeated BigQuery fields as arrays (#3480) * Treat repeated BigQuery fields as arrays * handle untransformed field types and None --- redash/query_runner/big_query.py | 44 ++++++++++++++++++-------------- 1 file changed, 25 insertions(+), 19 deletions(-) diff --git a/redash/query_runner/big_query.py b/redash/query_runner/big_query.py index 6ed63021b5..41eaae5045 100644 --- a/redash/query_runner/big_query.py +++ b/redash/query_runner/big_query.py @@ -32,28 +32,31 @@ } +def transform_cell(field_type, cell_value): + if cell_value is None: + return None + if field_type == 'INTEGER': + return int(cell_value) + elif field_type == 'FLOAT': + return float(cell_value) + elif field_type == 'BOOLEAN': + return cell_value.lower() == "true" + elif field_type == 'TIMESTAMP': + return datetime.datetime.fromtimestamp(float(cell_value)) + return cell_value + + def transform_row(row, fields): - column_index = 0 row_data = {} - for cell in row["f"]: + for column_index, cell in enumerate(row["f"]): field = fields[column_index] - cell_value = cell['v'] - - if cell_value is None: - pass - # Otherwise just cast the value - elif field['type'] == 'INTEGER': - cell_value = int(cell_value) - elif field['type'] == 'FLOAT': - cell_value = float(cell_value) - elif field['type'] == 'BOOLEAN': - cell_value = cell_value.lower() == "true" - elif field['type'] == 'TIMESTAMP': - cell_value = datetime.datetime.fromtimestamp(float(cell_value)) + if field['mode'] == 'REPEATED': + cell_value = [transform_cell(field['type'], item['v']) for item in cell['v']] + else: + cell_value = transform_cell(field['type'], cell['v']) row_data[field["name"]] = cell_value - column_index += 1 return row_data @@ -221,9 +224,12 @@ def _get_query_result(self, jobs, query): query_reply = jobs.getQueryResults(**query_result_request).execute() - columns = [{'name': f["name"], - 'friendly_name': f["name"], - 'type': types_map.get(f['type'], "string")} for f in query_reply["schema"]["fields"]] + columns = [{ + 'name': f["name"], + 'friendly_name': f["name"], + 'type': "string" if f['mode'] == "REPEATED" + else types_map.get(f['type'], "string") + } for f in query_reply["schema"]["fields"]] data = { "columns": columns, From fe4a7b65e70b356989a330fea3e17993216f5997 Mon Sep 17 00:00:00 2001 From: Ran Byron Date: Thu, 28 Mar 2019 05:55:03 +0200 Subject: [PATCH 018/179] Widget resize tests (#3620) --- client/app/components/dashboards/widget.html | 2 +- client/app/components/parameters.html | 2 +- client/app/pages/dashboards/dashboard.html | 4 +- .../integration/dashboard/dashboard_spec.js | 205 +++++++++++++++++- 4 files changed, 203 insertions(+), 10 deletions(-) diff --git a/client/app/components/dashboards/widget.html b/client/app/components/dashboards/widget.html index 94954841f7..ab8bf5f912 100644 --- a/client/app/components/dashboards/widget.html +++ b/client/app/components/dashboards/widget.html @@ -55,7 +55,7 @@
- + diff --git a/client/app/components/parameters.html b/client/app/components/parameters.html index 73eb15cee0..88762ae58b 100644 --- a/client/app/components/parameters.html +++ b/client/app/components/parameters.html @@ -3,7 +3,7 @@ ui-sortable="{ 'ui-floating': true, 'disabled': !editable }" ng-model="parameters" > -
+
), + (), + ] : [ + (), + ( + + ), + ]} + > +
+ + {currentStep === StepEnum.CONFIGURE_IT ? ( + Type Selection} + className="clickable" + onClick={this.resetType} + /> + ) : ()} + + + + {currentStep === StepEnum.SELECT_TYPE && this.renderTypeSelector()} + {currentStep !== StepEnum.SELECT_TYPE && this.renderForm()} +
+ + ); + } +} + +export default wrapDialog(CreateSourceDialog); diff --git a/client/app/components/HelpTrigger.jsx b/client/app/components/HelpTrigger.jsx index 2de707fb3a..9d9e464bca 100644 --- a/client/app/components/HelpTrigger.jsx +++ b/client/app/components/HelpTrigger.jsx @@ -12,7 +12,8 @@ import './HelpTrigger.less'; const DOMAIN = 'https://redash.io'; const HELP_PATH = '/help'; const IFRAME_TIMEOUT = 20000; -const TYPES = { + +export const TYPES = { HOME: [ '', 'Help', @@ -25,16 +26,50 @@ const TYPES = { '/user-guide/dashboards/sharing-dashboards', 'Guide: Sharing and Embedding Dashboards', ], + DS_ATHENA: [ + '/data-sources/amazon-athena-setup', + 'Guide: Help Setting up Amazon Athena', + ], + DS_BIGQUERY: [ + '/data-sources/bigquery-setup', + 'Guide: Help Setting up BigQuery', + ], + DS_URL: [ + '/data-sources/querying-urls', + 'Guide: Help Setting up URL', + ], + DS_MONGODB: [ + '/data-sources/mongodb-setup', + 'Guide: Help Setting up MongoDB', + ], + DS_GOOGLE_SPREADSHEETS: [ + '/data-sources/querying-a-google-spreadsheet', + 'Guide: Help Setting up Google Spreadsheets', + ], + DS_GOOGLE_ANALYTICS: [ + '/data-sources/google-analytics-setup', + 'Guide: Help Setting up Google Analytics', + ], + DS_AXIBASETSD: [ + '/data-sources/axibase-time-series-database', + 'Guide: Help Setting up Axibase Time Series', + ], + DS_RESULTS: [ + '/user-guide/querying/query-results-data-source', + 'Guide: Help Setting up Query Results', + ], }; export class HelpTrigger extends React.Component { static propTypes = { type: PropTypes.oneOf(Object.keys(TYPES)).isRequired, className: PropTypes.string, + children: PropTypes.node, } static defaultProps = { className: null, + children: , }; iframeRef = null @@ -92,7 +127,7 @@ export class HelpTrigger extends React.Component { - + {this.props.children} - Logo/Avatar + Logo/Avatar
{title}
{body &&
{body}
} @@ -20,12 +27,14 @@ PreviewCard.propTypes = { imageUrl: PropTypes.string.isRequired, title: PropTypes.node.isRequired, body: PropTypes.node, + roundedImage: PropTypes.bool, className: PropTypes.string, children: PropTypes.node, }; PreviewCard.defaultProps = { body: null, + roundedImage: true, className: '', children: null, }; diff --git a/client/app/components/cards-list/CardsList.jsx b/client/app/components/cards-list/CardsList.jsx new file mode 100644 index 0000000000..df4eb94ee2 --- /dev/null +++ b/client/app/components/cards-list/CardsList.jsx @@ -0,0 +1,86 @@ +import Card from 'antd/lib/card'; +import Input from 'antd/lib/input'; +import List from 'antd/lib/list'; +import { includes, isEmpty } from 'lodash'; +import PropTypes from 'prop-types'; +import React from 'react'; +import EmptyState from '@/components/items-list/components/EmptyState'; + +import './CardsList.less'; + +const { Search } = Input; +const { Meta } = Card; + +export default class CardsList extends React.Component { + static propTypes = { + items: PropTypes.arrayOf( + PropTypes.shape({ + title: PropTypes.string.isRequired, + imgSrc: PropTypes.string.isRequired, + onClick: PropTypes.func, + href: PropTypes.string, + }), + ), + showSearch: PropTypes.bool, + }; + + static defaultProps = { + items: [], + showSearch: false, + }; + + state = { + searchText: '', + }; + + // eslint-disable-next-line class-methods-use-this + renderListItem(item) { + const card = ( + {item.title}
)} + onClick={item.onClick} + hoverable + > + {item.title})} /> + + ); + return ( + + {item.href ? ({card}) : card} + + ); + } + + render() { + const { items, showSearch } = this.props; + const { searchText } = this.state; + + const filteredItems = items.filter(item => isEmpty(searchText) || + includes(item.title.toLowerCase(), searchText.toLowerCase())); + + return ( +
+ {showSearch && ( +
+
+ this.setState({ searchText: e.target.value })} + autoFocus + /> +
+
+ )} + {isEmpty(filteredItems) ? () : ( + this.renderListItem(item)} + /> + )} +
+ ); + } +} diff --git a/client/app/components/cards-list/CardsList.less b/client/app/components/cards-list/CardsList.less new file mode 100644 index 0000000000..e0a34544b5 --- /dev/null +++ b/client/app/components/cards-list/CardsList.less @@ -0,0 +1,22 @@ +.cards-list { + .cards-list-item { + text-align: center; + } + + .cards-list-item img { + margin-top: 10px; + width: 64px; + height: 64px; + } + + .cards-list-item h3 { + font-size: 13px; + height: 44px; + white-space: normal; + overflow: hidden; + /* autoprefixer: off */ + display: -webkit-box; + -webkit-line-clamp: 3; + -webkit-box-orient: vertical; + } +} diff --git a/client/app/components/dynamic-form/DynamicForm.jsx b/client/app/components/dynamic-form/DynamicForm.jsx index 8bfe38bdad..79d1fab5db 100644 --- a/client/app/components/dynamic-form/DynamicForm.jsx +++ b/client/app/components/dynamic-form/DynamicForm.jsx @@ -7,10 +7,9 @@ import Checkbox from 'antd/lib/checkbox'; import Button from 'antd/lib/button'; import Upload from 'antd/lib/upload'; import Icon from 'antd/lib/icon'; +import { includes, isFunction } from 'lodash'; import Select from 'antd/lib/select'; import notification from '@/services/notification'; -import { includes } from 'lodash'; -import { react2angular } from 'react2angular'; import { Field, Action, AntdForm } from '../proptypes'; import helper from './dynamicFormHelper'; @@ -26,8 +25,9 @@ const fieldRules = ({ type, required, minLength }) => { ].filter(rule => rule); }; -export const DynamicForm = Form.create()(class DynamicForm extends React.Component { +class DynamicForm extends React.Component { static propTypes = { + id: PropTypes.string, fields: PropTypes.arrayOf(Field), actions: PropTypes.arrayOf(Action), feedbackIcons: PropTypes.bool, @@ -38,6 +38,7 @@ export const DynamicForm = Form.create()(class DynamicForm extends React.Compone }; static defaultProps = { + id: null, fields: [], actions: [], feedbackIcons: false, @@ -179,14 +180,12 @@ export const DynamicForm = Form.create()(class DynamicForm extends React.Compone renderFields() { return this.props.fields.map((field) => { - const [firstField] = this.props.fields; const FormItem = Form.Item; - const { name, title, type, readOnly } = field; + const { name, title, type, readOnly, autoFocus, contentAfter } = field; const fieldLabel = title || helper.toHuman(name); - const { feedbackIcons } = this.props; + const { feedbackIcons, form } = this.props; const formItemProps = { - key: name, className: 'm-b-10', hasFeedback: type !== 'checkbox' && type !== 'file' && feedbackIcons, label: type === 'checkbox' ? '' : fieldLabel, @@ -194,16 +193,21 @@ export const DynamicForm = Form.create()(class DynamicForm extends React.Compone const fieldProps = { ...field.props, - autoFocus: (firstField === field), className: 'w-100', name, type, readOnly, + autoFocus, placeholder: field.placeholder, 'data-test': fieldLabel, }; - return ({this.renderField(field, fieldProps)}); + return ( + + {this.renderField(field, fieldProps)} + {isFunction(contentAfter) ? contentAfter(form.getFieldValue(name)) : contentAfter} + + ); }); } @@ -234,47 +238,17 @@ export const DynamicForm = Form.create()(class DynamicForm extends React.Compone disabled: this.state.isSubmitting, loading: this.state.isSubmitting, }; - const { hideSubmitButton, saveText } = this.props; + const { id, hideSubmitButton, saveText } = this.props; const saveButton = !hideSubmitButton; return ( -
+ {this.renderFields()} {saveButton && } {this.renderActions()}
); } -}); - -export default function init(ngModule) { - ngModule.component('dynamicForm', react2angular((props) => { - const fields = helper.getFields(props.type.configuration_schema, props.target); - - const onSubmit = (values, onSuccess, onError) => { - helper.updateTargetWithValues(props.target, values); - props.target.$save( - () => { - onSuccess('Saved.'); - }, - (error) => { - if (error.status === 400 && 'message' in error.data) { - onError(error.data.message); - } else { - onError('Failed saving.'); - } - }, - ); - }; - - const updatedProps = { - fields, - actions: props.target.id ? props.actions : [], - feedbackIcons: true, - onSubmit, - }; - return (); - }, ['target', 'type', 'actions'])); } -init.init = true; +export default Form.create()(DynamicForm); diff --git a/client/app/components/dynamic-form/dynamicFormHelper.js b/client/app/components/dynamic-form/dynamicFormHelper.js index 3eb10af6bb..c1d17997d3 100644 --- a/client/app/components/dynamic-form/dynamicFormHelper.js +++ b/client/app/components/dynamic-form/dynamicFormHelper.js @@ -1,3 +1,4 @@ +import React from 'react'; import { each, includes, isUndefined } from 'lodash'; function orderedInputs(properties, order, targetOptions) { @@ -57,10 +58,12 @@ function setDefaultValueForCheckboxes(configurationSchema, options = {}) { } } -function getFields(configurationSchema, target = {}) { +function getFields(type = {}, target = { options: {} }) { + const configurationSchema = type.configuration_schema; normalizeSchema(configurationSchema); setDefaultValueForCheckboxes(configurationSchema, target.options); + const isNewTarget = !target.id; const inputs = [ { name: 'name', @@ -68,6 +71,9 @@ function getFields(configurationSchema, target = {}) { type: 'text', required: true, initialValue: target.name, + contentAfter: React.createElement('hr'), + placeholder: `My ${type.name}`, + autoFocus: isNewTarget, }, ...orderedInputs(configurationSchema.properties, configurationSchema.order, target.options), ]; diff --git a/client/app/components/proptypes.js b/client/app/components/proptypes.js index 43f368cadf..05b585904d 100644 --- a/client/app/components/proptypes.js +++ b/client/app/components/proptypes.js @@ -55,8 +55,10 @@ export const Field = PropTypes.shape({ mode: PropTypes.string, required: PropTypes.bool, readOnly: PropTypes.bool, + autoFocus: PropTypes.bool, minLength: PropTypes.number, placeholder: PropTypes.string, + contentAfter: PropTypes.oneOfType([PropTypes.node, PropTypes.func]), loading: PropTypes.bool, props: PropTypes.object, // eslint-disable-line react/forbid-prop-types }); diff --git a/client/app/components/type-picker.html b/client/app/components/type-picker.html deleted file mode 100644 index 4ffe2c1259..0000000000 --- a/client/app/components/type-picker.html +++ /dev/null @@ -1,20 +0,0 @@ -
-

{{$ctrl.title}}

- -
-
-
- -
-
-
- -
-
-
- {{type.name}} -

{{type.name}}

-
-
-
-
diff --git a/client/app/components/type-picker.js b/client/app/components/type-picker.js deleted file mode 100644 index 8c2d57b6d9..0000000000 --- a/client/app/components/type-picker.js +++ /dev/null @@ -1,18 +0,0 @@ -import template from './type-picker.html'; - -export default function init(ngModule) { - ngModule.component('typePicker', { - template, - bindings: { - types: '<', - title: '@', - imgRoot: '@', - onTypeSelect: '=', - }, - controller() { - this.filter = {}; - }, - }); -} - -init.init = true; diff --git a/client/app/components/users/CreateUserDialog.jsx b/client/app/components/users/CreateUserDialog.jsx index ef799a92f2..dc630b8edc 100644 --- a/client/app/components/users/CreateUserDialog.jsx +++ b/client/app/components/users/CreateUserDialog.jsx @@ -2,7 +2,7 @@ import React from 'react'; import PropTypes from 'prop-types'; import Modal from 'antd/lib/modal'; import Alert from 'antd/lib/alert'; -import { DynamicForm } from '@/components/dynamic-form/DynamicForm'; +import DynamicForm from '@/components/dynamic-form/DynamicForm'; import { wrap as wrapDialog, DialogPropType } from '@/components/DialogWrapper'; import recordEvent from '@/services/recordEvent'; @@ -38,7 +38,7 @@ class CreateUserDialog extends React.Component { render() { const { savingUser, errorMessage } = this.state; const formFields = [ - { name: 'name', title: 'Name', type: 'text' }, + { name: 'name', title: 'Name', type: 'text', autoFocus: true }, { name: 'email', title: 'Email', type: 'email' }, ].map(field => ({ required: true, props: { onPressEnter: this.createUser }, ...field })); diff --git a/client/app/components/users/UserEdit.jsx b/client/app/components/users/UserEdit.jsx index 4b3bcd50c9..63489a6aec 100644 --- a/client/app/components/users/UserEdit.jsx +++ b/client/app/components/users/UserEdit.jsx @@ -10,7 +10,7 @@ import { Group } from '@/services/group'; import { currentUser } from '@/services/auth'; import { absoluteUrl } from '@/services/utils'; import { UserProfile } from '../proptypes'; -import { DynamicForm } from '../dynamic-form/DynamicForm'; +import DynamicForm from '../dynamic-form/DynamicForm'; import ChangePasswordDialog from './ChangePasswordDialog'; import InputWithCopy from '../InputWithCopy'; diff --git a/client/app/pages/data-sources/DataSourcesList.jsx b/client/app/pages/data-sources/DataSourcesList.jsx new file mode 100644 index 0000000000..29d51c2263 --- /dev/null +++ b/client/app/pages/data-sources/DataSourcesList.jsx @@ -0,0 +1,145 @@ +import React from 'react'; +import Button from 'antd/lib/button'; +import { react2angular } from 'react2angular'; +import { isEmpty, get } from 'lodash'; +import settingsMenu from '@/services/settingsMenu'; +import { DataSource, IMG_ROOT } from '@/services/data-source'; +import { policy } from '@/services/policy'; +import navigateTo from '@/services/navigateTo'; +import { $route } from '@/services/ng'; +import { routesToAngularRoutes } from '@/lib/utils'; +import CardsList from '@/components/cards-list/CardsList'; +import LoadingState from '@/components/items-list/components/LoadingState'; +import CreateSourceDialog from '@/components/CreateSourceDialog'; +import helper from '@/components/dynamic-form/dynamicFormHelper'; + +class DataSourcesList extends React.Component { + state = { + dataSourceTypes: [], + dataSources: [], + loading: true, + }; + + componentDidMount() { + Promise.all([ + DataSource.query().$promise, + DataSource.types().$promise, + ]).then(values => this.setState({ + dataSources: values[0], + dataSourceTypes: values[1], + loading: false, + }, () => { // all resources are loaded in state + if ($route.current.locals.isNewDataSourcePage) { + if (policy.canCreateDataSource()) { + this.showCreateSourceDialog(); + } else { + navigateTo('/data_sources'); + } + } + })); + } + + createDataSource = (selectedType, values) => { + const target = { options: {}, type: selectedType.type }; + helper.updateTargetWithValues(target, values); + + return DataSource.save(target).$promise.then((dataSource) => { + this.setState({ loading: true }); + DataSource.query(dataSources => this.setState({ dataSources, loading: false })); + return dataSource; + }).catch((error) => { + if (!(error instanceof Error)) { + error = new Error(get(error, 'data.message', 'Failed saving.')); + } + return Promise.reject(error); + }); + }; + + showCreateSourceDialog = () => { + CreateSourceDialog.showModal({ + types: this.state.dataSourceTypes, + sourceType: 'Data Source', + imageFolder: IMG_ROOT, + helpTriggerPrefix: 'DS_', + onCreate: this.createDataSource, + }).result.then((result = {}) => { + if (result.success) { + navigateTo(`data_sources/${result.data.id}`); + } + }); + }; + + renderDataSources() { + const { dataSources } = this.state; + const items = dataSources.map(dataSource => ({ + title: dataSource.name, + imgSrc: `${IMG_ROOT}/${dataSource.type}.png`, + href: `data_sources/${dataSource.id}`, + })); + + return isEmpty(dataSources) ? ( +
+ There are no data sources yet. + {policy.isCreateDataSourceEnabled() && ( +
+ Click here to add one. +
+ )} +
+ ) : (); + } + + render() { + const newDataSourceProps = { + type: 'primary', + onClick: policy.isCreateDataSourceEnabled() ? this.showCreateSourceDialog : null, + disabled: !policy.isCreateDataSourceEnabled(), + }; + + return ( +
+
+ +
+ {this.state.loading ? : this.renderDataSources()} +
+ ); + } +} + +export default function init(ngModule) { + settingsMenu.add({ + permission: 'admin', + title: 'Data Sources', + path: 'data_sources', + order: 1, + }); + + ngModule.component('pageDataSourcesList', react2angular(DataSourcesList)); + + return routesToAngularRoutes([ + { + path: '/data_sources', + title: 'Data Sources', + key: 'data_sources', + }, + { + path: '/data_sources/new', + title: 'Data Sources', + key: 'data_sources', + isNewDataSourcePage: true, + }, + ], { + template: '', + controller($scope, $exceptionHandler) { + 'ngInject'; + + $scope.handleError = $exceptionHandler; + }, + }); +} + +init.init = true; diff --git a/client/app/pages/data-sources/EditDataSource.jsx b/client/app/pages/data-sources/EditDataSource.jsx new file mode 100644 index 0000000000..75bf14b75b --- /dev/null +++ b/client/app/pages/data-sources/EditDataSource.jsx @@ -0,0 +1,152 @@ +import React from 'react'; +import PropTypes from 'prop-types'; +import { get, find, toUpper } from 'lodash'; +import { react2angular } from 'react2angular'; +import Modal from 'antd/lib/modal'; +import { DataSource, IMG_ROOT } from '@/services/data-source'; +import navigateTo from '@/services/navigateTo'; +import { $route } from '@/services/ng'; +import notification from '@/services/notification'; +import PromiseRejectionError from '@/lib/promise-rejection-error'; +import LoadingState from '@/components/items-list/components/LoadingState'; +import DynamicForm from '@/components/dynamic-form/DynamicForm'; +import helper from '@/components/dynamic-form/dynamicFormHelper'; +import { HelpTrigger, TYPES as HELP_TRIGGER_TYPES } from '@/components/HelpTrigger'; + +class EditDataSource extends React.Component { + static propTypes = { + onError: PropTypes.func, + }; + + static defaultProps = { + onError: () => {}, + }; + + state = { + dataSource: null, + type: null, + loading: true, + }; + + componentDidMount() { + DataSource.get({ id: $route.current.params.dataSourceId }).$promise.then((dataSource) => { + const { type } = dataSource; + this.setState({ dataSource }); + DataSource.types(types => this.setState({ type: find(types, { type }), loading: false })); + }).catch((error) => { + // ANGULAR_REMOVE_ME This code is related to Angular's HTTP services + if (error.status && error.data) { + error = new PromiseRejectionError(error); + } + this.props.onError(error); + }); + } + + saveDataSource = (values, successCallback, errorCallback) => { + const { dataSource } = this.state; + helper.updateTargetWithValues(dataSource, values); + dataSource.$save( + () => successCallback('Saved.'), + (error) => { + const message = get(error, 'data.message', 'Failed saving.'); + errorCallback(message); + }, + ); + } + + deleteDataSource = (callback) => { + const { dataSource } = this.state; + + const doDelete = () => { + dataSource.$delete(() => { + notification.success('Data source deleted successfully.'); + navigateTo('/data_sources', true); + }, () => { + callback(); + }); + }; + + Modal.confirm({ + title: 'Delete Data Source', + content: 'Are you sure you want to delete this data source?', + okText: 'Delete', + okType: 'danger', + onOk: doDelete, + onCancel: callback, + maskClosable: true, + autoFocusButton: null, + }); + }; + + testConnection = (callback) => { + const { dataSource } = this.state; + DataSource.test({ id: dataSource.id }, (httpResponse) => { + if (httpResponse.ok) { + notification.success('Success'); + } else { + notification.error('Connection Test Failed:', httpResponse.message, { duration: 10 }); + } + callback(); + }, () => { + notification.error('Connection Test Failed:', 'Unknown error occurred while performing connection test. Please try again later.', { duration: 10 }); + callback(); + }); + }; + + renderForm() { + const { dataSource, type } = this.state; + const fields = helper.getFields(type, dataSource); + const helpTriggerType = `DS_${toUpper(type.type)}`; + const formProps = { + fields, + type, + actions: [ + { name: 'Delete', type: 'danger', callback: this.deleteDataSource }, + { name: 'Test Connection', pullRight: true, callback: this.testConnection, disableWhenDirty: true }, + ], + onSubmit: this.saveDataSource, + feedbackIcons: true, + }; + + return ( +
+
+ {HELP_TRIGGER_TYPES[helpTriggerType] && ( + + Setup Instructions + + )} +
+
+ {type.name} +

{type.name}

+
+
+ +
+
+ ); + } + + render() { + return this.state.loading ? : this.renderForm(); + } +} + +export default function init(ngModule) { + ngModule.component('pageEditDataSource', react2angular(EditDataSource)); + + return { + '/data_sources/:dataSourceId': { + template: '', + title: 'Data Sources', + controller($scope, $exceptionHandler) { + 'ngInject'; + + $scope.handleError = $exceptionHandler; + }, + }, + }; +} + +init.init = true; diff --git a/client/app/pages/data-sources/list.html b/client/app/pages/data-sources/list.html deleted file mode 100644 index 56af90e071..0000000000 --- a/client/app/pages/data-sources/list.html +++ /dev/null @@ -1,17 +0,0 @@ - -
-
-

- - New Data Source - -

-
- - {{dataSource.name}} -

{{dataSource.name}}

- -
-
-
- \ No newline at end of file diff --git a/client/app/pages/data-sources/list.js b/client/app/pages/data-sources/list.js deleted file mode 100644 index ac8ff4487b..0000000000 --- a/client/app/pages/data-sources/list.js +++ /dev/null @@ -1,31 +0,0 @@ -import settingsMenu from '@/services/settingsMenu'; -import { policy } from '@/services/policy'; -import template from './list.html'; - -function DataSourcesCtrl(DataSource) { - this.policy = policy; - this.dataSources = DataSource.query(); -} - -export default function init(ngModule) { - settingsMenu.add({ - permission: 'admin', - title: 'Data Sources', - path: 'data_sources', - order: 1, - }); - - ngModule.component('dsListPage', { - controller: DataSourcesCtrl, - template, - }); - - return { - '/data_sources': { - template: '', - title: 'Data Sources', - }, - }; -} - -init.init = true; diff --git a/client/app/pages/data-sources/show.html b/client/app/pages/data-sources/show.html deleted file mode 100644 index 5883654482..0000000000 --- a/client/app/pages/data-sources/show.html +++ /dev/null @@ -1,37 +0,0 @@ - -
- - -
-
-
- ← Change -
- -
- {{type.name}} -

{{type.name}}

-
- -
-

- - Help - -

-
-
- -
- -
- - -
-
-
- diff --git a/client/app/pages/data-sources/show.js b/client/app/pages/data-sources/show.js deleted file mode 100644 index efd96c2510..0000000000 --- a/client/app/pages/data-sources/show.js +++ /dev/null @@ -1,133 +0,0 @@ -import { find } from 'lodash'; -import debug from 'debug'; -import template from './show.html'; -import notification from '@/services/notification'; - -const logger = debug('redash:http'); -export const deleteConfirm = { class: 'btn-warning', title: 'Delete' }; -export function logAndNotifyError(deleteObject, httpResponse) { - logger('Failed to delete ' + deleteObject + ': ', httpResponse.status, httpResponse.statusText, httpResponse.data); - notification.error('Failed to delete ' + deleteObject + '.'); -} -export function notifySuccessAndPath(deleteObject, deletePath, $location) { - notification.success(deleteObject + ' deleted successfully.'); - $location.path('/' + deletePath + '/'); -} - -function DataSourceCtrl( - $scope, $route, $routeParams, $http, $location, - currentUser, AlertDialog, DataSource, -) { - $scope.dataSource = $route.current.locals.dataSource; - $scope.dataSourceId = $routeParams.dataSourceId; - $scope.types = $route.current.locals.types; - $scope.type = find($scope.types, { type: $scope.dataSource.type }); - $scope.canChangeType = $scope.dataSource.id === undefined; - - $scope.helpLinks = { - athena: 'https://redash.io/help/data-sources/amazon-athena-setup', - bigquery: 'https://redash.io/help/data-sources/bigquery-setup', - url: 'https://redash.io/help/data-sources/querying-urls', - mongodb: 'https://redash.io/help/data-sources/mongodb-setup', - google_spreadsheets: 'https://redash.io/help/data-sources/querying-a-google-spreadsheet', - google_analytics: 'https://redash.io/help/data-sources/google-analytics-setup', - axibasetsd: 'https://redash.io/help/data-sources/axibase-time-series-database', - results: 'https://redash.io/help/user-guide/querying/query-results-data-source', - }; - - $scope.$watch('dataSource.id', (id) => { - if (id !== $scope.dataSourceId && id !== undefined) { - $location.path(`/data_sources/${id}`).replace(); - } - }); - - $scope.setType = (type) => { - $scope.type = type; - $scope.dataSource.type = type.type; - }; - - $scope.resetType = () => { - $scope.type = undefined; - $scope.dataSource = new DataSource({ options: {} }); - }; - - function deleteDataSource(callback) { - const doDelete = () => { - $scope.dataSource.$delete(() => { - notifySuccessAndPath('Data source', 'data_sources', $location); - }, (httpResponse) => { - logAndNotifyError('data source', httpResponse); - }); - }; - - const deleteTitle = 'Delete Data source'; - const deleteMessage = `Are you sure you want to delete the "${$scope.dataSource.name}" data source?`; - - AlertDialog.open(deleteTitle, deleteMessage, deleteConfirm).then(doDelete, callback); - } - - function testConnection(callback) { - DataSource.test({ id: $scope.dataSource.id }, (httpResponse) => { - if (httpResponse.ok) { - notification.success('Success'); - } else { - notification.error('Connection Test Failed:', httpResponse.message, { duration: 10 }); - } - callback(); - }, (httpResponse) => { - logger('Failed to test data source: ', httpResponse.status, httpResponse.statusText, httpResponse); - notification.error('Connection Test Failed:', 'Unknown error occurred while performing connection test. Please try again later.', { duration: 10 }); - callback(); - }); - } - - $scope.actions = [ - { name: 'Delete', type: 'danger', callback: deleteDataSource }, - { - name: 'Test Connection', pullRight: true, callback: testConnection, disableWhenDirty: true, - }, - ]; -} - -export default function init(ngModule) { - ngModule.controller('DataSourceCtrl', DataSourceCtrl); - - return { - '/data_sources/new': { - template, - controller: 'DataSourceCtrl', - title: 'Datasources', - resolve: { - dataSource: (DataSource) => { - 'ngInject'; - - return new DataSource({ options: {} }); - }, - types: ($http) => { - 'ngInject'; - - return $http.get('api/data_sources/types').then(response => response.data); - }, - }, - }, - '/data_sources/:dataSourceId': { - template, - controller: 'DataSourceCtrl', - title: 'Datasources', - resolve: { - dataSource: (DataSource, $route) => { - 'ngInject'; - - return DataSource.get({ id: $route.current.params.dataSourceId }).$promise; - }, - types: ($http) => { - 'ngInject'; - - return $http.get('api/data_sources/types').then(response => response.data); - }, - }, - }, - }; -} - -init.init = true; diff --git a/client/app/pages/destinations/DestinationsList.jsx b/client/app/pages/destinations/DestinationsList.jsx new file mode 100644 index 0000000000..2f68efddfb --- /dev/null +++ b/client/app/pages/destinations/DestinationsList.jsx @@ -0,0 +1,144 @@ +import React from 'react'; +import Button from 'antd/lib/button'; +import { react2angular } from 'react2angular'; +import { isEmpty, get } from 'lodash'; +import settingsMenu from '@/services/settingsMenu'; +import { Destination, IMG_ROOT } from '@/services/destination'; +import { policy } from '@/services/policy'; +import navigateTo from '@/services/navigateTo'; +import { $route } from '@/services/ng'; +import { routesToAngularRoutes } from '@/lib/utils'; +import CardsList from '@/components/cards-list/CardsList'; +import LoadingState from '@/components/items-list/components/LoadingState'; +import CreateSourceDialog from '@/components/CreateSourceDialog'; +import helper from '@/components/dynamic-form/dynamicFormHelper'; + +class DestinationsList extends React.Component { + state = { + destinationTypes: [], + destinations: [], + loading: true, + }; + + componentDidMount() { + Promise.all([ + Destination.query().$promise, + Destination.types().$promise, + ]).then(values => this.setState({ + destinations: values[0], + destinationTypes: values[1], + loading: false, + }, () => { // all resources are loaded in state + if ($route.current.locals.isNewDestinationPage) { + if (policy.canCreateDestination()) { + this.showCreateSourceDialog(); + } else { + navigateTo('/destinations'); + } + } + })); + } + + createDestination = (selectedType, values) => { + const target = { options: {}, type: selectedType.type }; + helper.updateTargetWithValues(target, values); + + return Destination.save(target).$promise.then((destination) => { + this.setState({ loading: true }); + Destination.query(destinations => this.setState({ destinations, loading: false })); + return destination; + }).catch((error) => { + if (!(error instanceof Error)) { + error = new Error(get(error, 'data.message', 'Failed saving.')); + } + return Promise.reject(error); + }); + }; + + showCreateSourceDialog = () => { + CreateSourceDialog.showModal({ + types: this.state.destinationTypes, + sourceType: 'Alert Destination', + imageFolder: IMG_ROOT, + onCreate: this.createDestination, + }).result.then((result = {}) => { + if (result.success) { + navigateTo(`destinations/${result.data.id}`); + } + }); + }; + + renderDestinations() { + const { destinations } = this.state; + const items = destinations.map(destination => ({ + title: destination.name, + imgSrc: `${IMG_ROOT}/${destination.type}.png`, + href: `destinations/${destination.id}`, + })); + + return isEmpty(destinations) ? ( +
+ There are no alert destinations yet. + {policy.isCreateDestinationEnabled() && ( +
+ Click here to add one. +
+ )} +
+ ) : (); + } + + render() { + const newDestinationProps = { + type: 'primary', + onClick: policy.isCreateDestinationEnabled() ? this.showCreateSourceDialog : null, + disabled: !policy.isCreateDestinationEnabled(), + }; + + return ( +
+
+ +
+ {this.state.loading ? : this.renderDestinations()} +
+ ); + } +} + +export default function init(ngModule) { + settingsMenu.add({ + permission: 'admin', + title: 'Alert Destinations', + path: 'destinations', + order: 4, + }); + + ngModule.component('pageDestinationsList', react2angular(DestinationsList)); + + return routesToAngularRoutes([ + { + path: '/destinations', + title: 'Alert Destinations', + key: 'destinations', + }, + { + path: '/destinations/new', + title: 'Alert Destinations', + key: 'destinations', + isNewDestinationPage: true, + }, + ], { + template: '', + controller($scope, $exceptionHandler) { + 'ngInject'; + + $scope.handleError = $exceptionHandler; + }, + }); +} + +init.init = true; diff --git a/client/app/pages/destinations/EditDestination.jsx b/client/app/pages/destinations/EditDestination.jsx new file mode 100644 index 0000000000..86677d3c12 --- /dev/null +++ b/client/app/pages/destinations/EditDestination.jsx @@ -0,0 +1,127 @@ +import React from 'react'; +import PropTypes from 'prop-types'; +import { get, find } from 'lodash'; +import { react2angular } from 'react2angular'; +import Modal from 'antd/lib/modal'; +import { Destination, IMG_ROOT } from '@/services/destination'; +import navigateTo from '@/services/navigateTo'; +import { $route } from '@/services/ng'; +import notification from '@/services/notification'; +import PromiseRejectionError from '@/lib/promise-rejection-error'; +import LoadingState from '@/components/items-list/components/LoadingState'; +import DynamicForm from '@/components/dynamic-form/DynamicForm'; +import helper from '@/components/dynamic-form/dynamicFormHelper'; + +class EditDestination extends React.Component { + static propTypes = { + onError: PropTypes.func, + }; + + static defaultProps = { + onError: () => {}, + }; + + state = { + destination: null, + type: null, + loading: true, + }; + + componentDidMount() { + Destination.get({ id: $route.current.params.destinationId }).$promise.then((destination) => { + const { type } = destination; + this.setState({ destination }); + Destination.types(types => this.setState({ type: find(types, { type }), loading: false })); + }).catch((error) => { + // ANGULAR_REMOVE_ME This code is related to Angular's HTTP services + if (error.status && error.data) { + error = new PromiseRejectionError(error); + } + this.props.onError(error); + }); + } + + saveDestination = (values, successCallback, errorCallback) => { + const { destination } = this.state; + helper.updateTargetWithValues(destination, values); + destination.$save( + () => successCallback('Saved.'), + (error) => { + const message = get(error, 'data.message', 'Failed saving.'); + errorCallback(message); + }, + ); + } + + deleteDestination = (callback) => { + const { destination } = this.state; + + const doDelete = () => { + destination.$delete(() => { + notification.success('Alert destination deleted successfully.'); + navigateTo('/destinations', true); + }, () => { + callback(); + }); + }; + + Modal.confirm({ + title: 'Delete Alert Destination', + content: 'Are you sure you want to delete this alert destination?', + okText: 'Delete', + okType: 'danger', + onOk: doDelete, + onCancel: callback, + maskClosable: true, + autoFocusButton: null, + }); + }; + + renderForm() { + const { destination, type } = this.state; + const fields = helper.getFields(type, destination); + const formProps = { + fields, + type, + actions: [ + { name: 'Delete', type: 'danger', callback: this.deleteDestination }, + ], + onSubmit: this.saveDestination, + feedbackIcons: true, + }; + + return ( +
+
+ {type.name} +

{type.name}

+
+
+ +
+
+ ); + } + + render() { + return this.state.loading ? : this.renderForm(); + } +} + +export default function init(ngModule) { + ngModule.component('pageEditDestination', react2angular(EditDestination)); + + return { + '/destinations/:destinationId': { + template: '', + title: 'Alert Destinations', + controller($scope, $exceptionHandler) { + 'ngInject'; + + $scope.handleError = $exceptionHandler; + }, + }, + }; +} + +init.init = true; diff --git a/client/app/pages/destinations/list.html b/client/app/pages/destinations/list.html deleted file mode 100644 index a3dcb2650f..0000000000 --- a/client/app/pages/destinations/list.html +++ /dev/null @@ -1,15 +0,0 @@ - -
-
-

- New Alert Destination -

-
- - {{destination.name}} -

{{destination.name}}

- -
-
-
- diff --git a/client/app/pages/destinations/list.js b/client/app/pages/destinations/list.js deleted file mode 100644 index bcb6ec2b5c..0000000000 --- a/client/app/pages/destinations/list.js +++ /dev/null @@ -1,27 +0,0 @@ -import settingsMenu from '@/services/settingsMenu'; -import template from './list.html'; - -function DestinationsCtrl($scope, $location, currentUser, Destination) { - $scope.destinations = Destination.query(); -} - -export default function init(ngModule) { - settingsMenu.add({ - permission: 'admin', - title: 'Alert Destinations', - path: 'destinations', - order: 4, - }); - - ngModule.controller('DestinationsCtrl', DestinationsCtrl); - - return { - '/destinations': { - template, - controller: 'DestinationsCtrl', - title: 'Destinations', - }, - }; -} - -init.init = true; diff --git a/client/app/pages/destinations/show.html b/client/app/pages/destinations/show.html deleted file mode 100644 index 51eaddbdfe..0000000000 --- a/client/app/pages/destinations/show.html +++ /dev/null @@ -1,29 +0,0 @@ - -
- - -
-
-
- ← Change Type -
- -
- {{type.name}} -

{{type.name}}

-
-
- -
- -
- - -
-
-
- diff --git a/client/app/pages/destinations/show.js b/client/app/pages/destinations/show.js deleted file mode 100644 index 699052b4a6..0000000000 --- a/client/app/pages/destinations/show.js +++ /dev/null @@ -1,92 +0,0 @@ -import { find } from 'lodash'; -import template from './show.html'; -import { deleteConfirm, logAndNotifyError, notifySuccessAndPath } from '../data-sources/show'; - -function DestinationCtrl( - $scope, $route, $routeParams, $http, $location, - currentUser, AlertDialog, Destination, -) { - $scope.destination = $route.current.locals.destination; - $scope.destinationId = $routeParams.destinationId; - $scope.types = $route.current.locals.types; - $scope.type = find($scope.types, { type: $scope.destination.type }); - $scope.canChangeType = $scope.destination.id === undefined; - - $scope.$watch('destination.id', (id) => { - if (id !== $scope.destinationId && id !== undefined) { - $location.path(`/destinations/${id}`).replace(); - } - }); - - $scope.setType = (type) => { - $scope.type = type; - $scope.destination.type = type.type; - }; - - $scope.resetType = () => { - $scope.type = undefined; - $scope.destination = new Destination({ options: {} }); - }; - - function deleteDestination(callback) { - const doDelete = () => { - $scope.destination.$delete(() => { - notifySuccessAndPath('Destination', 'destinations', $location); - }, (httpResponse) => { - logAndNotifyError('destination', httpResponse); - }); - }; - - const title = 'Delete Destination'; - const message = `Are you sure you want to delete the "${$scope.destination.name}" destination?`; - - AlertDialog.open(title, message, deleteConfirm).then(doDelete, callback); - } - - $scope.actions = [ - { name: 'Delete', type: 'danger', callback: deleteDestination }, - ]; -} - -export default function init(ngModule) { - ngModule.controller('DestinationCtrl', DestinationCtrl); - - return { - '/destinations/new': { - template, - controller: 'DestinationCtrl', - title: 'Destinations', - resolve: { - destination: (Destination) => { - 'ngInject'; - - return new Destination({ options: {} }); - }, - types: ($http) => { - 'ngInject'; - - return $http.get('api/destinations/types').then(response => response.data); - }, - }, - }, - '/destinations/:destinationId': { - template, - controller: 'DestinationCtrl', - title: 'Destinations', - resolve: { - destination: (Destination, $route) => { - 'ngInject'; - - return Destination.get({ id: $route.current.params.destinationId }).$promise; - }, - types: ($http) => { - 'ngInject'; - - return $http.get('api/destinations/types').then(response => response.data); - }, - }, - }, - }; -} - -init.init = true; diff --git a/client/app/services/data-source.js b/client/app/services/data-source.js index f1d69d5cbc..efd4422a0d 100644 --- a/client/app/services/data-source.js +++ b/client/app/services/data-source.js @@ -1,8 +1,10 @@ export const SCHEMA_NOT_SUPPORTED = 1; export const SCHEMA_LOAD_ERROR = 2; +export const IMG_ROOT = '/static/images/db-logos'; export let DataSource = null; // eslint-disable-line import/no-mutable-exports + function DataSourceService($q, $resource, $http) { function fetchSchema(dataSourceId, refresh = false) { const params = {}; @@ -17,6 +19,13 @@ function DataSourceService($q, $resource, $http) { const actions = { get: { method: 'GET', cache: false, isArray: false }, query: { method: 'GET', cache: false, isArray: true }, + save: { method: 'POST' }, + types: { + method: 'GET', + cache: false, + isArray: true, + url: 'api/data_sources/types', + }, test: { method: 'POST', cache: false, diff --git a/client/app/services/destination.js b/client/app/services/destination.js index 9c3c6d2f61..6452ceb0ef 100644 --- a/client/app/services/destination.js +++ b/client/app/services/destination.js @@ -1,10 +1,19 @@ +export const IMG_ROOT = '/static/images/destinations'; + export let Destination = null; // eslint-disable-line import/no-mutable-exports function DestinationService($resource) { const actions = { get: { method: 'GET', cache: false, isArray: false }, + types: { + method: 'GET', + cache: false, + isArray: true, + url: 'api/destinations/types', + }, query: { method: 'GET', cache: false, isArray: true }, }; + return $resource('api/destinations/:id', { id: '@id' }, actions); } diff --git a/client/app/services/policy/DefaultPolicy.js b/client/app/services/policy/DefaultPolicy.js index b1101addbe..9aa1ed3046 100644 --- a/client/app/services/policy/DefaultPolicy.js +++ b/client/app/services/policy/DefaultPolicy.js @@ -17,6 +17,14 @@ export default class DefaultPolicy { return currentUser.isAdmin; } + canCreateDestination() { + return currentUser.isAdmin; + } + + isCreateDestinationEnabled() { + return currentUser.isAdmin; + } + canCreateDashboard() { return currentUser.hasPermission('create_dashboard'); } diff --git a/client/cypress/integration/data-source/create_data_source_spec.js b/client/cypress/integration/data-source/create_data_source_spec.js index fe0c3370e6..315935d591 100644 --- a/client/cypress/integration/data-source/create_data_source_spec.js +++ b/client/cypress/integration/data-source/create_data_source_spec.js @@ -5,18 +5,21 @@ describe('Create Data Source', () => { }); it('renders the page and takes a screenshot', () => { - cy.getByTestId('TypePicker').should('contain', 'PostgreSQL'); + cy.getByTestId('CreateSourceDialog').should('contain', 'PostgreSQL'); + cy.wait(1000); // eslint-disable-line cypress/no-unnecessary-waiting cy.percySnapshot('Create Data Source - Types'); }); it('creates a new PostgreSQL data source', () => { - cy.getByTestId('TypePicker').contains('PostgreSQL').click(); + cy.getByTestId('SearchSource').type('PostgreSQL'); + cy.getByTestId('CreateSourceDialog').contains('PostgreSQL').click(); cy.getByTestId('Name').type('Redash'); - cy.getByTestId('Host').type('{selectall}postgres'); + cy.getByTestId('Host').type('postgres'); cy.getByTestId('User').type('postgres'); cy.getByTestId('Password').type('postgres'); cy.getByTestId('Database Name').type('postgres{enter}'); + cy.getByTestId('CreateSourceButton').click(); cy.contains('Saved.'); }); diff --git a/client/cypress/integration/destination/create_destination_spec.js b/client/cypress/integration/destination/create_destination_spec.js index 3c82a316dd..7b6bf3c09e 100644 --- a/client/cypress/integration/destination/create_destination_spec.js +++ b/client/cypress/integration/destination/create_destination_spec.js @@ -5,7 +5,8 @@ describe('Create Destination', () => { }); it('renders the page and takes a screenshot', () => { - cy.getByTestId('TypePicker').should('contain', 'Email'); + cy.getByTestId('CreateSourceDialog').should('contain', 'Email'); + cy.wait(1000); // eslint-disable-line cypress/no-unnecessary-waiting cy.percySnapshot('Create Destination - Types'); }); }); From 1871287a1f8224df7e40c82d9ef7d3ecf5a9a44b Mon Sep 17 00:00:00 2001 From: Ran Byron Date: Thu, 28 Mar 2019 10:08:13 +0200 Subject: [PATCH 020/179] Fixed notification alignment (#3645) --- client/app/assets/less/ant.less | 8 ++++---- client/app/assets/less/inc/ant-variables.less | 6 ++++++ 2 files changed, 10 insertions(+), 4 deletions(-) diff --git a/client/app/assets/less/ant.less b/client/app/assets/less/ant.less index 4d27fb4d99..b422c638ea 100644 --- a/client/app/assets/less/ant.less +++ b/client/app/assets/less/ant.less @@ -254,9 +254,9 @@ // flexible width for notifications .@{notification-prefix-cls} { - width: auto; - - &-notice { - padding-right: 48px; + // vertical centering + &-notice-close { + top: 20px; + right: 20px; } } diff --git a/client/app/assets/less/inc/ant-variables.less b/client/app/assets/less/inc/ant-variables.less index 4e86e0b8b2..428990094a 100644 --- a/client/app/assets/less/inc/ant-variables.less +++ b/client/app/assets/less/inc/ant-variables.less @@ -72,3 +72,9 @@ @table-row-hover-bg: fade(@redash-gray, 5%); @table-padding-vertical: 7px; @table-padding-horizontal: 10px; + +/* -------------------------------------------------------- + Notification +-----------------------------------------------------------*/ +@notification-padding: @notification-padding-vertical 48px @notification-padding-vertical 17px; +@notification-width: auto; \ No newline at end of file From ec4f77c8b7fbcb4b04821e98bc12a0f994a6b16e Mon Sep 17 00:00:00 2001 From: Omer Lachish Date: Thu, 28 Mar 2019 15:01:06 +0200 Subject: [PATCH 021/179] Change has_access and require_access signatures (#3611) * change has_access and require_access signatures to work with the objects that require access, instead of their groups * rename `object` to `obj` * support both objects and group dicts in `has_access` and `require_access` * simplify permission tests once `has_access` accepts groups --- redash/handlers/alerts.py | 8 ++++---- redash/handlers/data_sources.py | 2 +- redash/handlers/favorites.py | 4 ++-- redash/handlers/queries.py | 8 ++++---- redash/handlers/query_results.py | 6 +++--- redash/handlers/widgets.py | 2 +- redash/models/parameterized_query.py | 2 +- redash/permissions.py | 12 +++++++----- redash/query_runner/query_results.py | 2 +- redash/serializers.py | 2 +- 10 files changed, 25 insertions(+), 23 deletions(-) diff --git a/redash/handlers/alerts.py b/redash/handlers/alerts.py index cc6bc9f1f1..8dd3ec1c8e 100644 --- a/redash/handlers/alerts.py +++ b/redash/handlers/alerts.py @@ -14,7 +14,7 @@ class AlertResource(BaseResource): def get(self, alert_id): alert = get_object_or_404(models.Alert.get_by_id_and_org, alert_id, self.current_org) - require_access(alert.groups, self.current_user, view_only) + require_access(alert, self.current_user, view_only) self.record_event({ 'action': 'view', 'object_id': alert.id, @@ -53,7 +53,7 @@ def post(self): query = models.Query.get_by_id_and_org(req['query_id'], self.current_org) - require_access(query.groups, self.current_user, view_only) + require_access(query, self.current_user, view_only) alert = models.Alert( name=req['name'], @@ -89,7 +89,7 @@ def post(self, alert_id): req = request.get_json(True) alert = models.Alert.get_by_id_and_org(alert_id, self.current_org) - require_access(alert.groups, self.current_user, view_only) + require_access(alert, self.current_user, view_only) kwargs = {'alert': alert, 'user': self.current_user} if 'destination_id' in req: @@ -113,7 +113,7 @@ def post(self, alert_id): def get(self, alert_id): alert_id = int(alert_id) alert = models.Alert.get_by_id_and_org(alert_id, self.current_org) - require_access(alert.groups, self.current_user, view_only) + require_access(alert, self.current_user, view_only) subscriptions = models.AlertSubscription.all(alert_id) return [s.to_dict() for s in subscriptions] diff --git a/redash/handlers/data_sources.py b/redash/handlers/data_sources.py index 3862d1a6ef..1324d092dd 100644 --- a/redash/handlers/data_sources.py +++ b/redash/handlers/data_sources.py @@ -144,7 +144,7 @@ def post(self): class DataSourceSchemaResource(BaseResource): def get(self, data_source_id): data_source = get_object_or_404(models.DataSource.get_by_id_and_org, data_source_id, self.current_org) - require_access(data_source.groups, self.current_user, view_only) + require_access(data_source, self.current_user, view_only) refresh = request.args.get('refresh') is not None response = {} diff --git a/redash/handlers/favorites.py b/redash/handlers/favorites.py index 627c75bf73..1c2eff32b3 100644 --- a/redash/handlers/favorites.py +++ b/redash/handlers/favorites.py @@ -10,7 +10,7 @@ class QueryFavoriteResource(BaseResource): def post(self, query_id): query = get_object_or_404(models.Query.get_by_id_and_org, query_id, self.current_org) - require_access(query.groups, self.current_user, view_only) + require_access(query, self.current_user, view_only) fav = models.Favorite(org_id=self.current_org.id, object=query, user=self.current_user) models.db.session.add(fav) @@ -31,7 +31,7 @@ def post(self, query_id): def delete(self, query_id): query = get_object_or_404(models.Query.get_by_id_and_org, query_id, self.current_org) - require_access(query.groups, self.current_user, view_only) + require_access(query, self.current_user, view_only) models.Favorite.query.filter( models.Favorite.object_id == query_id, diff --git a/redash/handlers/queries.py b/redash/handlers/queries.py index b18acc1bf4..d204e51eff 100644 --- a/redash/handlers/queries.py +++ b/redash/handlers/queries.py @@ -220,7 +220,7 @@ def post(self): """ query_def = request.get_json(force=True) data_source = models.DataSource.get_by_id_and_org(query_def.pop('data_source_id'), self.current_org) - require_access(data_source.groups, self.current_user, not_view_only) + require_access(data_source, self.current_user, not_view_only) require_access_to_dropdown_queries(self.current_user, query_def) for field in ['id', 'created_at', 'api_key', 'visualizations', 'latest_query_data', 'last_modified_by']: @@ -356,7 +356,7 @@ def get(self, query_id): Responds with the :ref:`query ` contents. """ q = get_object_or_404(models.Query.get_by_id_and_org, query_id, self.current_org) - require_access(q.groups, self.current_user, view_only) + require_access(q, self.current_user, view_only) result = QuerySerializer(q, with_visualizations=True).serialize() result['can_edit'] = can_modify(q, self.current_user) @@ -393,7 +393,7 @@ def post(self, query_id): Responds with created :ref:`query ` object. """ query = get_object_or_404(models.Query.get_by_id_and_org, query_id, self.current_org) - require_access(query.data_source.groups, self.current_user, not_view_only) + require_access(query.data_source, self.current_user, not_view_only) forked_query = query.fork(self.current_user) models.db.session.commit() @@ -422,7 +422,7 @@ def post(self, query_id): abort(403, message="Please use a user API key.") query = get_object_or_404(models.Query.get_by_id_and_org, query_id, self.current_org) - require_access(query.groups, self.current_user, not_view_only) + require_access(query, self.current_user, not_view_only) parameter_values = collect_parameters_from_request(request.args) parameterized_query = ParameterizedQuery(query.query_text) diff --git a/redash/handlers/query_results.py b/redash/handlers/query_results.py index 436a3edfb7..a4bfc8d073 100644 --- a/redash/handlers/query_results.py +++ b/redash/handlers/query_results.py @@ -135,7 +135,7 @@ def post(self): data_source = models.DataSource.get_by_id_and_org(params.get('data_source_id'), self.current_org) - if not has_access(data_source.groups, self.current_user, not_view_only): + if not has_access(data_source, self.current_user, not_view_only): return {'job': {'status': 4, 'error': 'You do not have permission to run queries with this data source.'}}, 403 self.record_event({ @@ -213,7 +213,7 @@ def post(self, query_id): allow_executing_with_view_only_permissions = query.parameterized.is_safe - if has_access(query.data_source.groups, self.current_user, allow_executing_with_view_only_permissions): + if has_access(query.data_source, self.current_user, allow_executing_with_view_only_permissions): return run_query(query.parameterized, parameters, query.data_source, query_id, max_age) else: return {'job': {'status': 4, 'error': 'You do not have permission to run queries with this data source.'}}, 403 @@ -264,7 +264,7 @@ def get(self, query_id=None, query_result_id=None, filetype='json'): abort(404, message='No cached result found for this query.') if query_result: - require_access(query_result.data_source.groups, self.current_user, view_only) + require_access(query_result.data_source, self.current_user, view_only) if isinstance(self.current_user, models.ApiUser): event = { diff --git a/redash/handlers/widgets.py b/redash/handlers/widgets.py index ef81d7cb88..0949cc59fb 100644 --- a/redash/handlers/widgets.py +++ b/redash/handlers/widgets.py @@ -34,7 +34,7 @@ def post(self): visualization_id = widget_properties.pop('visualization_id') if visualization_id: visualization = models.Visualization.get_by_id_and_org(visualization_id, self.current_org) - require_access(visualization.query_rel.groups, self.current_user, view_only) + require_access(visualization.query_rel, self.current_user, view_only) else: visualization = None diff --git a/redash/models/parameterized_query.py b/redash/models/parameterized_query.py index 7dc955025f..d3ff757f0f 100644 --- a/redash/models/parameterized_query.py +++ b/redash/models/parameterized_query.py @@ -23,7 +23,7 @@ def _load_result(query_id, should_require_access): query = models.Query.get_by_id_and_org(query_id, current_org) if should_require_access: - require_access(query.data_source.groups, current_user, view_only) + require_access(query.data_source, current_user, view_only) query_result = models.QueryResult.get_by_id_and_org(query.latest_query_data_id, current_org) diff --git a/redash/permissions.py b/redash/permissions.py index d8d2b8df7d..7118a6b83d 100644 --- a/redash/permissions.py +++ b/redash/permissions.py @@ -14,24 +14,26 @@ ACCESS_TYPES = (ACCESS_TYPE_VIEW, ACCESS_TYPE_MODIFY, ACCESS_TYPE_DELETE) -def has_access(object_groups, user, need_view_only): +def has_access(obj, user, need_view_only): + groups = obj.groups if hasattr(obj, 'groups') else obj + if 'admin' in user.permissions: return True - matching_groups = set(object_groups.keys()).intersection(user.group_ids) + matching_groups = set(groups.keys()).intersection(user.group_ids) if not matching_groups: return False required_level = 1 if need_view_only else 2 - group_level = 1 if all(flatten([object_groups[group] for group in matching_groups])) else 2 + group_level = 1 if all(flatten([groups[group] for group in matching_groups])) else 2 return required_level <= group_level -def require_access(object_groups, user, need_view_only): - if not has_access(object_groups, user, need_view_only): +def require_access(obj, user, need_view_only): + if not has_access(obj, user, need_view_only): abort(403) diff --git a/redash/query_runner/query_results.py b/redash/query_runner/query_results.py index 2e8b95432f..502ede831a 100644 --- a/redash/query_runner/query_results.py +++ b/redash/query_runner/query_results.py @@ -38,7 +38,7 @@ def _load_query(user, query_id): if user.org_id != query.org_id: raise PermissionError("Query id {} not found.".format(query.id)) - if not has_access(query.data_source.groups, user, not_view_only): + if not has_access(query.data_source, user, not_view_only): raise PermissionError(u"You are not allowed to execute queries on {} data source (used for query id {}).".format( query.data_source.name, query.id)) diff --git a/redash/serializers.py b/redash/serializers.py index f1f01f3313..81e1df38d2 100644 --- a/redash/serializers.py +++ b/redash/serializers.py @@ -194,7 +194,7 @@ def serialize_dashboard(obj, with_widgets=False, user=None, with_favorite_state= for w in obj.widgets: if w.visualization_id is None: widgets.append(serialize_widget(w)) - elif user and has_access(w.visualization.query_rel.groups, user, view_only): + elif user and has_access(w.visualization.query_rel, user, view_only): widgets.append(serialize_widget(w)) else: widget = project(serialize_widget(w), From 704b78a0039b89cbe8b216b2613c1a4e6ee23149 Mon Sep 17 00:00:00 2001 From: Levko Kravets Date: Thu, 28 Mar 2019 20:33:05 +0200 Subject: [PATCH 022/179] [Feature, Bug fix] Migrate Timer component to React; update TimeAgo component (#3648) --- client/app/components/TimeAgo.jsx | 106 ++++++++----------- client/app/components/Timer.jsx | 37 +++++++ client/app/components/dashboards/widget.html | 2 +- client/app/components/rd-timer.js | 32 ------ client/app/lib/hooks/useForceUpdate.js | 6 ++ client/app/pages/queries/query.html | 14 +-- 6 files changed, 97 insertions(+), 100 deletions(-) create mode 100644 client/app/components/Timer.jsx delete mode 100644 client/app/components/rd-timer.js create mode 100644 client/app/lib/hooks/useForceUpdate.js diff --git a/client/app/components/TimeAgo.jsx b/client/app/components/TimeAgo.jsx index 281f972188..4bc1ee62a5 100644 --- a/client/app/components/TimeAgo.jsx +++ b/client/app/components/TimeAgo.jsx @@ -1,83 +1,65 @@ import moment from 'moment'; import { isNil } from 'lodash'; -import React from 'react'; +import React, { useEffect } from 'react'; import ReactDOM from 'react-dom'; import PropTypes from 'prop-types'; import { Moment } from '@/components/proptypes'; import { clientConfig } from '@/services/auth'; +import useForceUpdate from '@/lib/hooks/useForceUpdate'; -const autoUpdateList = new Set(); - -function updateComponents() { - autoUpdateList.forEach(component => component.update()); - setTimeout(updateComponents, 30 * 1000); +function toMoment(value) { + value = !isNil(value) ? moment(value) : null; + return value && value.isValid() ? value : null; } -updateComponents(); - -export class TimeAgo extends React.PureComponent { - static propTypes = { - // `date` and `placeholder` used in `getDerivedStateFromProps` - // eslint-disable-next-line react/no-unused-prop-types - date: PropTypes.oneOfType([ - PropTypes.string, - PropTypes.number, - PropTypes.instanceOf(Date), - Moment, - ]), - // eslint-disable-next-line react/no-unused-prop-types - placeholder: PropTypes.string, - autoUpdate: PropTypes.bool, - }; - - static defaultProps = { - date: null, - placeholder: '', - autoUpdate: true, - }; - // Initial state, to get rid of React warning - state = { - title: null, - value: null, - }; +export function TimeAgo({ date, placeholder, autoUpdate }) { + const startDate = toMoment(date); - static getDerivedStateFromProps({ date, placeholder }) { - // if `date` prop is not empty and a valid date/time - convert it to `moment` - date = !isNil(date) ? moment(date) : null; - date = date && date.isValid() ? date : null; + const value = startDate ? startDate.fromNow() : placeholder; + const title = startDate ? startDate.format(clientConfig.dateTimeFormat) : ''; - return { - value: date ? date.fromNow() : placeholder, - title: date ? date.format(clientConfig.dateTimeFormat) : '', - }; - } + const forceUpdate = useForceUpdate(); - componentDidMount() { - autoUpdateList.add(this); - this.update(true); - } - - componentWillUnmount() { - autoUpdateList.delete(this); - } - - update(force = false) { - if (force || this.props.autoUpdate) { - this.setState(this.constructor.getDerivedStateFromProps(this.props)); + useEffect(() => { + if (autoUpdate) { + const timer = setInterval(forceUpdate, 30 * 1000); + return () => clearInterval(timer); } - } + }, [autoUpdate]); - render() { - return {this.state.value}; - } + return {value}; } +TimeAgo.propTypes = { + // `date` and `placeholder` used in `getDerivedStateFromProps` + // eslint-disable-next-line react/no-unused-prop-types + date: PropTypes.oneOfType([ + PropTypes.string, + PropTypes.number, + PropTypes.instanceOf(Date), + Moment, + ]), + // eslint-disable-next-line react/no-unused-prop-types + placeholder: PropTypes.string, + autoUpdate: PropTypes.bool, +}; + +TimeAgo.defaultProps = { + date: null, + placeholder: '', + autoUpdate: true, +}; + export default function init(ngModule) { ngModule.directive('amTimeAgo', () => ({ - link($scope, element, attr) { + link($scope, $element, attr) { const modelName = attr.amTimeAgo; $scope.$watch(modelName, (value) => { - ReactDOM.render(, element[0]); + ReactDOM.render(, $element[0]); + }); + + $scope.$on('$destroy', () => { + ReactDOM.unmountComponentAtNode($element[0]); }); }, })); @@ -91,6 +73,10 @@ export default function init(ngModule) { // Initial render will occur here as well ReactDOM.render(, $element[0]); }); + + $scope.$on('$destroy', () => { + ReactDOM.unmountComponentAtNode($element[0]); + }); }, }); } diff --git a/client/app/components/Timer.jsx b/client/app/components/Timer.jsx new file mode 100644 index 0000000000..9e5e1e5651 --- /dev/null +++ b/client/app/components/Timer.jsx @@ -0,0 +1,37 @@ +import moment from 'moment'; +import { useMemo, useEffect } from 'react'; +import PropTypes from 'prop-types'; +import { react2angular } from 'react2angular'; +import { Moment } from '@/components/proptypes'; +import useForceUpdate from '@/lib/hooks/useForceUpdate'; + +export function Timer({ from }) { + const startTime = useMemo(() => moment(from).valueOf(), [from]); + const forceUpdate = useForceUpdate(); + + useEffect(() => { + const timer = setInterval(forceUpdate, 1000); + return () => clearInterval(timer); + }, []); + + return moment.utc(moment.now() - startTime).format('HH:mm:ss'); +} + +Timer.propTypes = { + from: PropTypes.oneOfType([ + PropTypes.string, + PropTypes.number, + PropTypes.instanceOf(Date), + Moment, + ]), +}; + +Timer.defaultProps = { + from: null, +}; + +export default function init(ngModule) { + ngModule.component('rdTimer', react2angular(Timer)); +} + +init.init = true; diff --git a/client/app/components/dashboards/widget.html b/client/app/components/dashboards/widget.html index ab8bf5f912..8314d31343 100644 --- a/client/app/components/dashboards/widget.html +++ b/client/app/components/dashboards/widget.html @@ -58,7 +58,7 @@ - + diff --git a/client/app/components/rd-timer.js b/client/app/components/rd-timer.js deleted file mode 100644 index 5c552292f5..0000000000 --- a/client/app/components/rd-timer.js +++ /dev/null @@ -1,32 +0,0 @@ -import moment from 'moment'; - -function rdTimer() { - return { - restrict: 'E', - scope: { timestamp: '=' }, - template: '{{currentTime}}', - controller($scope) { - $scope.currentTime = '00:00:00'; - - // We're using setInterval directly instead of $timeout, to avoid using $apply, to - // prevent the digest loop being run every second. - let currentTimer = setInterval(() => { - $scope.currentTime = moment(moment() - moment($scope.timestamp)).utc().format('HH:mm:ss'); - $scope.$digest(); - }, 1000); - - $scope.$on('$destroy', () => { - if (currentTimer) { - clearInterval(currentTimer); - currentTimer = null; - } - }); - }, - }; -} - -export default function init(ngModule) { - ngModule.directive('rdTimer', rdTimer); -} - -init.init = true; diff --git a/client/app/lib/hooks/useForceUpdate.js b/client/app/lib/hooks/useForceUpdate.js new file mode 100644 index 0000000000..fc02bc8db1 --- /dev/null +++ b/client/app/lib/hooks/useForceUpdate.js @@ -0,0 +1,6 @@ +import { useState } from 'react'; + +export default function useForceUpdate() { + const [, setValue] = useState(false); + return () => setValue(value => !value); +} diff --git a/client/app/pages/queries/query.html b/client/app/pages/queries/query.html index d95b0b76db..565ffa8836 100644 --- a/client/app/pages/queries/query.html +++ b/client/app/pages/queries/query.html @@ -196,23 +196,23 @@

-
+
Executing query… - +
-
+
Loading results… - +
-
+
Query in queue… - +
-
+
Error running query: {{queryResult.getError()}}
From 12782e4daf460aaa440ef492220fee75d51f3977 Mon Sep 17 00:00:00 2001 From: Gabriel Dutra Date: Fri, 29 Mar 2019 07:50:30 -0300 Subject: [PATCH 023/179] Fix Percy diff due to Api Key secret (#3654) --- client/cypress/integration/user/edit_profile_spec.js | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/client/cypress/integration/user/edit_profile_spec.js b/client/cypress/integration/user/edit_profile_spec.js index f93ff27bf8..cf6bc8239b 100644 --- a/client/cypress/integration/user/edit_profile_spec.js +++ b/client/cypress/integration/user/edit_profile_spec.js @@ -37,13 +37,11 @@ describe('Edit Profile', () => { }); it('renders the page and takes a screenshot', () => { + cy.getByTestId('Groups').should('contain', 'admin'); cy.getByTestId('ApiKey').then(($apiKey) => { $apiKey.val('secret'); + cy.percySnapshot('User Profile'); }); - - cy.getByTestId('Groups').should('contain', 'admin'); - - cy.percySnapshot('User Profile'); }); context('changing password', () => { From 02a58520726aaee8d549f6ecdff9e9254e1c1501 Mon Sep 17 00:00:00 2001 From: Ran Byron Date: Fri, 29 Mar 2019 21:47:26 +0300 Subject: [PATCH 024/179] Widget size and position test (#3628) --- .../integration/dashboard/dashboard_spec.js | 27 ++++++++++++++++--- 1 file changed, 24 insertions(+), 3 deletions(-) diff --git a/client/cypress/integration/dashboard/dashboard_spec.js b/client/cypress/integration/dashboard/dashboard_spec.js index 3d59b67589..ae7af1d310 100644 --- a/client/cypress/integration/dashboard/dashboard_spec.js +++ b/client/cypress/integration/dashboard/dashboard_spec.js @@ -16,15 +16,15 @@ function editDashboard() { }); } -function addTextboxByAPI(text, dashId) { +function addTextboxByAPI(text, dashId, options = {}) { const data = { width: 1, dashboard_id: dashId, visualization_id: null, text: 'text', - options: { + options: Object.assign({ position: { col: 0, row: 0, sizeX: 3, sizeY: 3 }, - }, + }, options), }; return cy.request('POST', 'api/widgets', data).then(({ body }) => { @@ -298,6 +298,27 @@ describe('Dashboard', () => { cy.get('@textboxEl').should('contain', newContent); }); }); + + it('renders textbox according to position configuration', function () { + const id = this.dashboardId; + const txb1Pos = { col: 0, row: 0, sizeX: 3, sizeY: 2 }; + const txb2Pos = { col: 1, row: 1, sizeX: 3, sizeY: 4 }; + + cy.viewport(1215, 800); + addTextboxByAPI('x', id, { position: txb1Pos }) + .then(() => addTextboxByAPI('x', id, { position: txb2Pos })) + .then((elTestId) => { + cy.visit(this.dashboardUrl); + return cy.getByTestId(elTestId); + }) + .should(($el) => { + const { top, left } = $el.offset(); + expect(top).to.eq(214); + expect(left).to.eq(215); + expect($el.width()).to.eq(585); + expect($el.height()).to.eq(185); + }); + }); }); describe('Grid compliant widgets', () => { From 33ad89a3811189ba98b2fcef672ee013dece2dae Mon Sep 17 00:00:00 2001 From: Omer Lachish Date: Mon, 1 Apr 2019 11:19:18 +0300 Subject: [PATCH 025/179] in case of a parameter type mismatch, show the actual message to the user (#3664) --- client/app/services/query-result.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/client/app/services/query-result.js b/client/app/services/query-result.js index 06fa301c9b..08c3aad885 100644 --- a/client/app/services/query-result.js +++ b/client/app/services/query-result.js @@ -69,7 +69,7 @@ function QueryResultService($resource, $timeout, $q, QueryResultError) { logger('Unknown error', response); queryResult.update({ job: { - error: 'unknown error occurred. Please try again later.', + error: response.data.message || 'unknown error occurred. Please try again later.', status: 4, }, }); From 1333aae7fb36909c494db5342b419b0b7d5f9a47 Mon Sep 17 00:00:00 2001 From: Omer Lachish Date: Mon, 1 Apr 2019 11:19:52 +0300 Subject: [PATCH 026/179] Handle dropdown queries which are detached from data source (#3453) * handle an edge case where dropdown queries are connected to data sources that no longer exist * Rethinking it, an empty result set makes no sense and it's better to throw an error * remove redundant import --- redash/models/parameterized_query.py | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/redash/models/parameterized_query.py b/redash/models/parameterized_query.py index d3ff757f0f..8f2dd263af 100644 --- a/redash/models/parameterized_query.py +++ b/redash/models/parameterized_query.py @@ -1,6 +1,7 @@ import pystache from functools import partial from flask_login import current_user +from flask_restful import abort from numbers import Number from redash.utils import mustache_render, json_loads from redash.permissions import require_access, view_only @@ -27,7 +28,12 @@ def _load_result(query_id, should_require_access): query_result = models.QueryResult.get_by_id_and_org(query.latest_query_data_id, current_org) - return json_loads(query_result.data) + if query.data_source: + require_access(query.data_source.groups, current_user, view_only) + query_result = models.QueryResult.get_by_id_and_org(query.latest_query_data_id, current_org) + return json_loads(query_result.data) + else: + abort(400, message="This query is detached from any data source. Please select a different query.") def dropdown_values(query_id, should_require_access=True): From 6f9aee42a78f461019104b35167da0340df50edc Mon Sep 17 00:00:00 2001 From: Justin Clift Date: Mon, 1 Apr 2019 19:21:06 +1100 Subject: [PATCH 027/179] Update to modern Redis for the docker images (#3640) --- setup/docker-compose.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup/docker-compose.yml b/setup/docker-compose.yml index e6b64ac951..aea6369b5d 100644 --- a/setup/docker-compose.yml +++ b/setup/docker-compose.yml @@ -33,7 +33,7 @@ services: QUEUES: "queries" WORKERS_COUNT: 2 redis: - image: redis:3.0-alpine + image: redis:5.0-alpine restart: always postgres: image: postgres:9.5-alpine From 5decd2624a9abb42987208ec2395cf327e6964d1 Mon Sep 17 00:00:00 2001 From: Ran Byron Date: Mon, 1 Apr 2019 13:49:24 +0300 Subject: [PATCH 028/179] Fixed wrong width assertion (#3665) --- client/cypress/integration/dashboard/dashboard_spec.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/client/cypress/integration/dashboard/dashboard_spec.js b/client/cypress/integration/dashboard/dashboard_spec.js index ae7af1d310..887264789b 100644 --- a/client/cypress/integration/dashboard/dashboard_spec.js +++ b/client/cypress/integration/dashboard/dashboard_spec.js @@ -315,7 +315,7 @@ describe('Dashboard', () => { const { top, left } = $el.offset(); expect(top).to.eq(214); expect(left).to.eq(215); - expect($el.width()).to.eq(585); + expect($el.width()).to.eq(600); expect($el.height()).to.eq(185); }); }); From dd477d49ec4b9447fe352c4e3a8f5035ab40a44a Mon Sep 17 00:00:00 2001 From: Omer Lachish Date: Tue, 2 Apr 2019 11:45:38 +0300 Subject: [PATCH 029/179] Sharing embeds with safe parameters (#3495) * change has_access and require_access signatures to work with the objects that require access, instead of their groups * change has_access and require_access signatures to work with the objects that require access, instead of their groups * use the textless endpoint (/api/queries/:id/results) for pristine queriest * Revert "use the textless endpoint (/api/queries/:id/results) for pristine" This reverts commit cd2cee77385ecf79efd2f1aa21fab0dd43943264. * go to textless /api/queries/:id/results by default * change `run_query`'s signature to accept a ParameterizedQuery instead of constructing it inside * raise HTTP 400 when receiving invalid parameter values. Fixes #3394 * support querystring params * extract coercing of numbers to function, along with a friendlier implementation * wire embeds to textless endpoint * allow users with view_only permissions to execute queries on the textless endpoint, as it only allows safe queries to run * enqueue jobs for ApiUsers * add parameters component for embeds * include existing parameters in embed code * fetch correct values for json requests * remove previous embed parameter code * rename `id` to `user_id` * support executing queries using Query api_keys by instantiating an ApiUser that would be able to execute the specific query * bring back ALLOW_PARAMETERS_IN_EMBEDS (with link on deprecation coming up) * show deprecation messages for ALLOW_PARAMETERS_IN_EMBEDS. Also, move other message (email not verified) to use the same mechanism * add link to forum message on setting deprecation * rephrase deprecation message * add link to forum message regarding embed deprecation * change API to /api/queries/:id/dropdowns/:dropdown_id * split to 2 different dropdown endpoints and implement the second * add test cases for /api/queries/:id/dropdowns/:id * use new /dropdowns endpoint in frontend * first e2e test for sharing embeds * Pleasing the CodeClimate overlords * All glory to CodeClimate * change has_access and require_access signatures to work with the objects that require access, instead of their groups * split has_access between normal users and ApiKey users * remove residues from bad rebase * allow access to safe queries via api keys * rename `object` to `obj` * support both objects and group dicts in `has_access` and `require_access` * simplify permission tests once `has_access` accepts groups * change has_access and require_access signatures to work with the objects that require access, instead of their groups * rename `object` to `obj` * support both objects and group dicts in `has_access` and `require_access` * simplify permission tests once `has_access` accepts groups * fix bad rebase * send embed parameters through POST data * no need to log `is_api_key` * move query fetching by api_key to within the Query model * fetch user by adding a get_by_id function on the User model * pass parameters as POST data (fixes test failure introduced by switching from query string parameters to POST data) * test the right thing - queries with safe parameters should be embeddable * introduce cy.clickThrough * add another Cypress test to make sure unsafe queries cannot be embedded * serialize Parameters into query string * set is_api_key as the last parameter to (hopefully) avoid backward-dependency problems * Update redash/models/parameterized_query.py Co-Authored-By: rauchy * attempt to fix empty percy snapshots * snap percies after DOM is fully loaded --- .../EditParameterSettingsDialog.jsx | 8 +-- client/app/components/ParameterValueInput.jsx | 2 +- client/app/components/QueryEditor.jsx | 8 ++- client/app/components/app-view/template.html | 2 +- client/app/components/parameters.html | 2 +- .../components/queries/embed-code-dialog.html | 2 +- .../components/queries/embed-code-dialog.js | 2 +- .../queries/visualization-embed.html | 6 +- .../components/queries/visualization-embed.js | 6 +- client/app/pages/home/home.html | 5 +- client/app/pages/home/index.js | 4 +- client/app/pages/queries/query.html | 2 +- client/app/services/auth.js | 5 +- client/app/services/query.js | 8 +++ .../integration/embed/share_embed_spec.js | 54 ++++++++++++++ client/cypress/support/commands.js | 8 +++ redash/handlers/authentication.py | 16 ++++- redash/handlers/queries.py | 4 +- redash/handlers/query_results.py | 70 ++++--------------- redash/models/__init__.py | 4 ++ redash/models/parameterized_query.py | 16 ++++- redash/models/users.py | 4 ++ redash/permissions.py | 11 +++ redash/settings/__init__.py | 9 +-- redash/tasks/queries.py | 28 +++++--- tests/handlers/test_embed.py | 29 -------- tests/handlers/test_query_results.py | 19 +++++ tests/models/test_parameterized_query.py | 8 +++ tests/tasks/test_queries.py | 12 ++-- 29 files changed, 227 insertions(+), 127 deletions(-) create mode 100644 client/cypress/integration/embed/share_embed_spec.js diff --git a/client/app/components/EditParameterSettingsDialog.jsx b/client/app/components/EditParameterSettingsDialog.jsx index 01b0504156..4f26ec9803 100644 --- a/client/app/components/EditParameterSettingsDialog.jsx +++ b/client/app/components/EditParameterSettingsDialog.jsx @@ -131,7 +131,7 @@ function EditParameterSettingsDialog(props) { footer={[( ), ( - )]} @@ -153,9 +153,9 @@ function EditParameterSettingsDialog(props) { /> - setParam({ ...param, type })} data-test="ParameterTypeSelect"> + +

)} /> - - ); return ( - - {item.href ? ({card}) : card} - + + {item.title} +

{item.title}

+ ); } render() { - const { items, showSearch } = this.props; + const { showSearch } = this.props; const { searchText } = this.state; - const filteredItems = items.filter(item => isEmpty(searchText) || + const filteredItems = this.items.filter(item => isEmpty(searchText) || includes(item.title.toLowerCase(), searchText.toLowerCase())); return ( -
+
{showSearch && (
@@ -73,12 +70,11 @@ export default class CardsList extends React.Component {
)} {isEmpty(filteredItems) ? () : ( - this.renderListItem(item)} - /> +
+
+ {filteredItems.map(item => this.renderListItem(item))} +
+
)}
); diff --git a/client/app/components/cards-list/CardsList.less b/client/app/components/cards-list/CardsList.less deleted file mode 100644 index e0a34544b5..0000000000 --- a/client/app/components/cards-list/CardsList.less +++ /dev/null @@ -1,22 +0,0 @@ -.cards-list { - .cards-list-item { - text-align: center; - } - - .cards-list-item img { - margin-top: 10px; - width: 64px; - height: 64px; - } - - .cards-list-item h3 { - font-size: 13px; - height: 44px; - white-space: normal; - overflow: hidden; - /* autoprefixer: off */ - display: -webkit-box; - -webkit-line-clamp: 3; - -webkit-box-orient: vertical; - } -} From b96094b878585421b5d8d598578c046b5d62af30 Mon Sep 17 00:00:00 2001 From: Omer Lachish Date: Sun, 14 Apr 2019 14:59:21 +0300 Subject: [PATCH 049/179] add a test to make sure reset password form are displayed correctly (#3678) --- redash/handlers/authentication.py | 4 ++-- tests/handlers/test_authentication.py | 7 +++++++ 2 files changed, 9 insertions(+), 2 deletions(-) diff --git a/redash/handlers/authentication.py b/redash/handlers/authentication.py index 9b1cd883af..cec7f67fdc 100644 --- a/redash/handlers/authentication.py +++ b/redash/handlers/authentication.py @@ -25,7 +25,7 @@ def get_google_auth_url(next_path): return google_auth_url -def render_token_login_page(template, org_slug, token, invite=True): +def render_token_login_page(template, org_slug, token, invite): try: user_id = validate_token(token) org = current_org._get_current_object() @@ -77,7 +77,7 @@ def render_token_login_page(template, org_slug, token, invite=True): @routes.route(org_scoped_rule('/invite/'), methods=['GET', 'POST']) def invite(token, org_slug=None): - return render_token_login_page("invite.html", org_slug, token) + return render_token_login_page("invite.html", org_slug, token, True) @routes.route(org_scoped_rule('/reset/'), methods=['GET', 'POST']) diff --git a/tests/handlers/test_authentication.py b/tests/handlers/test_authentication.py index 1559921d14..a4cd43463a 100644 --- a/tests/handlers/test_authentication.py +++ b/tests/handlers/test_authentication.py @@ -8,6 +8,13 @@ from redash.models import User +class TestResetPassword(BaseTestCase): + def test_shows_reset_password_form(self): + user = self.factory.create_user(is_invitation_pending=False) + token = invite_token(user) + response = self.get_request('/reset/{}'.format(token), org=self.factory.org) + self.assertEqual(response.status_code, 200) + class TestInvite(BaseTestCase): def test_expired_invite_token(self): From 5b30d081d7f0381f8ff7ab67e4c4978d9da37ef9 Mon Sep 17 00:00:00 2001 From: Omer Lachish Date: Mon, 15 Apr 2019 12:06:37 +0300 Subject: [PATCH 050/179] Dynamic query time limits (#3702) * extract time limit decisions to a dynamic settings function * introduce environment variable for scheduled query time limits * pass in org_id to query_time_limit * add an interaction test that verifies that time limits are applied to jobs * really important newlines according to CodeClimate --- redash/settings/__init__.py | 4 +++- redash/settings/dynamic_settings.py | 10 ++++++++++ redash/tasks/queries.py | 5 ++--- tests/tasks/test_queries.py | 10 ++++++++++ 4 files changed, 25 insertions(+), 4 deletions(-) create mode 100644 redash/settings/dynamic_settings.py diff --git a/redash/settings/__init__.py b/redash/settings/__init__.py index 1d5b3c431b..53e96aacd0 100644 --- a/redash/settings/__init__.py +++ b/redash/settings/__init__.py @@ -1,4 +1,5 @@ import os +import importlib from funcy import distinct, remove from flask_talisman import talisman @@ -288,7 +289,8 @@ def email_server_is_configured(): disabled_query_runners = array_from_string(os.environ.get("REDASH_DISABLED_QUERY_RUNNERS", "")) QUERY_RUNNERS = remove(set(disabled_query_runners), distinct(enabled_query_runners + additional_query_runners)) -ADHOC_QUERY_TIME_LIMIT = int_or_none(os.environ.get('REDASH_ADHOC_QUERY_TIME_LIMIT', None)) + +dynamic_settings = importlib.import_module(os.environ.get('REDASH_DYNAMIC_SETTINGS_MODULE', 'redash.settings.dynamic_settings')) # Destinations default_destinations = [ diff --git a/redash/settings/dynamic_settings.py b/redash/settings/dynamic_settings.py new file mode 100644 index 0000000000..fcd9494447 --- /dev/null +++ b/redash/settings/dynamic_settings.py @@ -0,0 +1,10 @@ +import os +from .helpers import int_or_none + + +# Replace this method with your own implementation in case you want to limit the time limit on certain queries or users. +def query_time_limit(is_scheduled, user_id, org_id): + scheduled_time_limit = int_or_none(os.environ.get('REDASH_SCHEDULED_QUERY_TIME_LIMIT', None)) + adhoc_time_limit = int_or_none(os.environ.get('REDASH_ADHOC_QUERY_TIME_LIMIT', None)) + + return scheduled_time_limit if is_scheduled else adhoc_time_limit diff --git a/redash/tasks/queries.py b/redash/tasks/queries.py index 1b0a6f6458..ab16209138 100644 --- a/redash/tasks/queries.py +++ b/redash/tasks/queries.py @@ -120,15 +120,12 @@ def enqueue_query(query, data_source, user_id, is_api_key=False, scheduled_query if not job: pipe.multi() - time_limit = None - if scheduled_query: queue_name = data_source.scheduled_queue_name scheduled_query_id = scheduled_query.id else: queue_name = data_source.queue_name scheduled_query_id = None - time_limit = settings.ADHOC_QUERY_TIME_LIMIT args = (query, data_source.id, metadata, user_id, scheduled_query_id, is_api_key) argsrepr = json_dumps({ @@ -140,6 +137,8 @@ def enqueue_query(query, data_source, user_id, is_api_key=False, scheduled_query 'user_id': user_id }) + time_limit = settings.dynamic_settings.query_time_limit(scheduled_query, user_id, data_source.org_id) + result = execute_query.apply_async(args=args, argsrepr=argsrepr, queue=queue_name, diff --git a/tests/tasks/test_queries.py b/tests/tasks/test_queries.py index c90e87a8f5..758d6e5402 100644 --- a/tests/tasks/test_queries.py +++ b/tests/tasks/test_queries.py @@ -28,6 +28,16 @@ def test_multiple_enqueue_of_same_query(self): self.assertEqual(1, execute_query.apply_async.call_count) + @mock.patch('redash.settings.dynamic_settings.query_time_limit', return_value=60) + def test_limits_query_time(self, _): + query = self.factory.create_query() + execute_query.apply_async = mock.MagicMock(side_effect=gen_hash) + + enqueue_query(query.query_text, query.data_source, query.user_id, False, query, {'Username': 'Arik', 'Query ID': query.id}) + + _, kwargs = execute_query.apply_async.call_args + self.assertEqual(60, kwargs.get('time_limit')) + def test_multiple_enqueue_of_different_query(self): query = self.factory.create_query() execute_query.apply_async = mock.MagicMock(side_effect=gen_hash) From e485c964c561a9f2e74bf82db1f0ddc46aaa6257 Mon Sep 17 00:00:00 2001 From: Arik Fraimovich Date: Mon, 15 Apr 2019 13:58:30 +0300 Subject: [PATCH 051/179] Add rate limits to user creation/update (#3709) * Add rate limits for user resources. * Disable rate limiting in tests (except for tests that need it). * Update strings to unicode to avoid SQLA warnings --- redash/handlers/users.py | 8 +++++++- tests/__init__.py | 5 +++-- tests/handlers/test_authentication.py | 4 +++- tests/models/test_queries.py | 24 ++++++++++++------------ 4 files changed, 25 insertions(+), 16 deletions(-) diff --git a/redash/handlers/users.py b/redash/handlers/users.py index 911bfe6488..0eca25bd01 100644 --- a/redash/handlers/users.py +++ b/redash/handlers/users.py @@ -9,7 +9,7 @@ from disposable_email_domains import blacklist from funcy import partial -from redash import models +from redash import models, limiter from redash.permissions import require_permission, require_admin_or_owner, is_admin_or_owner, \ require_permission_or_owner, require_admin from redash.handlers.base import BaseResource, require_fields, get_object_or_404, paginate, order_results as _order_results @@ -51,6 +51,9 @@ def invite_user(org, inviter, user, send_email=True): class UserListResource(BaseResource): + decorators = BaseResource.decorators + \ + [limiter.limit('200/day;50/hour', methods=['POST'])] + def get_users(self, disabled, pending, search_term): if disabled: users = models.User.all_disabled(self.current_org) @@ -190,6 +193,9 @@ def post(self, user_id): class UserResource(BaseResource): + decorators = BaseResource.decorators + \ + [limiter.limit('50/hour', methods=['POST'])] + def get(self, user_id): require_permission_or_owner('list_users', user_id) user = get_object_or_404(models.User.get_by_id_and_org, user_id, self.current_org) diff --git a/tests/__init__.py b/tests/__init__.py index 2119b1118f..07a8fe8f85 100644 --- a/tests/__init__.py +++ b/tests/__init__.py @@ -16,7 +16,7 @@ # Make sure rate limit is enabled os.environ['REDASH_RATELIMIT_ENABLED'] = "true" -from redash import create_app +from redash import create_app, limiter from redash import redis_connection from redash.models import db from redash.utils import json_dumps, json_loads @@ -48,6 +48,7 @@ def setUp(self): self.db = db self.app.config['TESTING'] = True self.app.config['SERVER_NAME'] = 'localhost' + limiter.enabled = False self.app_ctx = self.app.app_context() self.app_ctx.push() db.session.close() @@ -115,7 +116,7 @@ def post_request(self, path, data=None, org=None, headers=None): def assertResponseEqual(self, expected, actual): for k, v in expected.iteritems(): if isinstance(v, datetime.datetime) or isinstance(actual[k], - datetime.datetime): + datetime.datetime): continue if isinstance(v, list): diff --git a/tests/handlers/test_authentication.py b/tests/handlers/test_authentication.py index a4cd43463a..e868821170 100644 --- a/tests/handlers/test_authentication.py +++ b/tests/handlers/test_authentication.py @@ -3,7 +3,7 @@ import mock from tests import BaseTestCase -from redash import settings +from redash import settings, limiter from redash.authentication.account import invite_token from redash.models import User @@ -15,6 +15,7 @@ def test_shows_reset_password_form(self): response = self.get_request('/reset/{}'.format(token), org=self.factory.org) self.assertEqual(response.status_code, 200) + class TestInvite(BaseTestCase): def test_expired_invite_token(self): @@ -82,6 +83,7 @@ def test_valid_password(self): class TestLogin(BaseTestCase): def test_throttle_login(self): + limiter.enabled = True # Extract the limit from settings (ex: '50/day') limit = settings.THROTTLE_LOGIN_PATTERN.split('/')[0] for _ in range(0, int(limit)): diff --git a/tests/models/test_queries.py b/tests/models/test_queries.py index d90fcaca28..94d25c5d05 100644 --- a/tests/models/test_queries.py +++ b/tests/models/test_queries.py @@ -21,13 +21,13 @@ def create_tagged_query(self, tags): return query def test_all_tags(self): - self.create_tagged_query(tags=['tag1']) - self.create_tagged_query(tags=['tag1', 'tag2']) - self.create_tagged_query(tags=['tag1', 'tag2', 'tag3']) + self.create_tagged_query(tags=[u'tag1']) + self.create_tagged_query(tags=[u'tag1', u'tag2']) + self.create_tagged_query(tags=[u'tag1', u'tag2', u'tag3']) self.assertEqual( list(Query.all_tags(self.factory.user)), - [('tag1', 3), ('tag2', 2), ('tag3', 1)] + [(u'tag1', 3), (u'tag2', 2), (u'tag3', 1)] ) def test_search_finds_in_name(self): @@ -52,9 +52,9 @@ def test_search_finds_in_description(self): self.assertNotIn(q3, queries) def test_search_by_id_returns_query(self): - q1 = self.factory.create_query(description="Testing search") - q2 = self.factory.create_query(description="Testing searching") - q3 = self.factory.create_query(description="Testing sea rch") + q1 = self.factory.create_query(description=u"Testing search") + q2 = self.factory.create_query(description=u"Testing searching") + q3 = self.factory.create_query(description=u"Testing sea rch") db.session.flush() queries = Query.search(str(q3.id), [self.factory.default_group.id]) @@ -63,20 +63,20 @@ def test_search_by_id_returns_query(self): self.assertNotIn(q2, queries) def test_search_by_number(self): - q = self.factory.create_query(description="Testing search 12345") + q = self.factory.create_query(description=u"Testing search 12345") db.session.flush() queries = Query.search('12345', [self.factory.default_group.id]) self.assertIn(q, queries) def test_search_respects_groups(self): - other_group = Group(org=self.factory.org, name="Other Group") + other_group = Group(org=self.factory.org, name=u"Other Group") db.session.add(other_group) ds = self.factory.create_data_source(group=other_group) - q1 = self.factory.create_query(description="Testing search", data_source=ds) - q2 = self.factory.create_query(description="Testing searching") - q3 = self.factory.create_query(description="Testing sea rch") + q1 = self.factory.create_query(description=u"Testing search", data_source=ds) + q2 = self.factory.create_query(description=u"Testing searching") + q3 = self.factory.create_query(description=u"Testing sea rch") queries = list(Query.search("Testing", [self.factory.default_group.id])) From 9b3dd82ec079a5e68ededcb535e8a3b0ae6b7bed Mon Sep 17 00:00:00 2001 From: Arik Fraimovich Date: Wed, 17 Apr 2019 09:43:33 +0300 Subject: [PATCH 052/179] Sync PyAthena/botocore versions with requirements_all_ds.txt. (#3713) --- requirements_dev.txt | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/requirements_dev.txt b/requirements_dev.txt index 5c09af6dab..dc33df9a77 100644 --- a/requirements_dev.txt +++ b/requirements_dev.txt @@ -6,7 +6,7 @@ mock==2.0.0 # PyMongo and Athena dependencies are needed for some of the unit tests: # (this is not perfect and we should resolve this in a different way) pymongo[tls,srv]==3.6.1 -botocore==1.12.85 -PyAthena>=1.0.0 +botocore==1.12.115 +PyAthena>=1.5.0 ptvsd==4.2.3 freezegun==0.3.11 From 18761cf07b215b50eab17d1b26996886d5f12208 Mon Sep 17 00:00:00 2001 From: Ran Byron Date: Wed, 17 Apr 2019 10:07:48 +0300 Subject: [PATCH 053/179] Dashboard auto-saving (#3653) --- .../assets/less/redash/redash-newstyle.less | 4 + .../components/dashboards/gridstack/index.js | 73 +------------- client/app/pages/dashboards/dashboard.html | 18 ++-- client/app/pages/dashboards/dashboard.js | 68 ++++++-------- client/app/pages/dashboards/dashboard.less | 32 +++++++ client/app/services/widget.js | 6 ++ .../integration/dashboard/dashboard_spec.js | 94 +++---------------- 7 files changed, 100 insertions(+), 195 deletions(-) diff --git a/client/app/assets/less/redash/redash-newstyle.less b/client/app/assets/less/redash/redash-newstyle.less index 28a0389568..8808113960 100644 --- a/client/app/assets/less/redash/redash-newstyle.less +++ b/client/app/assets/less/redash/redash-newstyle.less @@ -937,4 +937,8 @@ text.slicetext { .markdown strong { font-weight: bold; +} + +.disabled-silent { + pointer-events: none; } \ No newline at end of file diff --git a/client/app/components/dashboards/gridstack/index.js b/client/app/components/dashboards/gridstack/index.js index cd1c16c85b..7189540f0d 100644 --- a/client/app/components/dashboards/gridstack/index.js +++ b/client/app/components/dashboards/gridstack/index.js @@ -69,6 +69,7 @@ function gridstack($parse, dashboardGridOptions) { scope: { editing: '=', batchUpdate: '=', // set by directive - for using in wrapper components + onLayoutChanged: '=', isOneColumnMode: '=', }, controller() { @@ -121,67 +122,6 @@ function gridstack($parse, dashboardGridOptions) { }); }; - this.batchUpdateWidgets = (items) => { - // This method is used to update multiple widgets with a single - // reflow (for example, restore positions when dashboard editing cancelled). - // "dirty" part of code: updating grid and DOM nodes directly. - // layout reflow is triggered by `batchUpdate`/`commit` calls - this.update((grid) => { - _.each(grid.grid.nodes, (node) => { - const item = items[node.id]; - if (item) { - if (_.isNumber(item.col)) { - node.x = parseFloat(item.col); - node.el.attr('data-gs-x', node.x); - node._dirty = true; - } - - if (_.isNumber(item.row)) { - node.y = parseFloat(item.row); - node.el.attr('data-gs-y', node.y); - node._dirty = true; - } - - if (_.isNumber(item.sizeX)) { - node.width = parseFloat(item.sizeX); - node.el.attr('data-gs-width', node.width); - node._dirty = true; - } - - if (_.isNumber(item.sizeY)) { - node.height = parseFloat(item.sizeY); - node.el.attr('data-gs-height', node.height); - node._dirty = true; - } - - if (_.isNumber(item.minSizeX)) { - node.minWidth = parseFloat(item.minSizeX); - node.el.attr('data-gs-min-width', node.minWidth); - node._dirty = true; - } - - if (_.isNumber(item.maxSizeX)) { - node.maxWidth = parseFloat(item.maxSizeX); - node.el.attr('data-gs-max-width', node.maxWidth); - node._dirty = true; - } - - if (_.isNumber(item.minSizeY)) { - node.minHeight = parseFloat(item.minSizeY); - node.el.attr('data-gs-min-height', node.minHeight); - node._dirty = true; - } - - if (_.isNumber(item.maxSizeY)) { - node.maxHeight = parseFloat(item.maxSizeY); - node.el.attr('data-gs-max-height', node.maxHeight); - node._dirty = true; - } - } - }); - }); - }; - this.removeWidget = ($element) => { const grid = this.grid(); if (grid) { @@ -235,9 +175,7 @@ function gridstack($parse, dashboardGridOptions) { }; }, link: ($scope, $element, $attr, controller) => { - const batchUpdateAssignable = _.isFunction($parse($attr.batchUpdate).assign); - const isOneColumnModeAssignable = _.isFunction($parse($attr.batchUpdate).assign); - + const isOneColumnModeAssignable = _.isFunction($parse($attr.onLayoutChanged).assign); let enablePolling = true; $element.addClass('grid-stack'); @@ -300,6 +238,9 @@ function gridstack($parse, dashboardGridOptions) { $(node.el).trigger('gridstack.changed', node); } }); + if ($scope.onLayoutChanged) { + $scope.onLayoutChanged(); + } changedNodes = {}; }); @@ -315,10 +256,6 @@ function gridstack($parse, dashboardGridOptions) { controller.setEditing(!!value); }); - if (batchUpdateAssignable) { - $scope.batchUpdate = controller.batchUpdateWidgets; - } - $scope.$on('$destroy', () => { enablePolling = false; controller.$el = null; diff --git a/client/app/pages/dashboards/dashboard.html b/client/app/pages/dashboards/dashboard.html index 9246b02c5e..cc987db29b 100644 --- a/client/app/pages/dashboards/dashboard.html +++ b/client/app/pages/dashboards/dashboard.html @@ -15,17 +15,15 @@

-
- +
+ Saving + Saved -
@@ -91,7 +89,7 @@

-
{ + const saveDashboardLayout = () => { if (!this.dashboard.canEdit()) { return; } + // calc diff, bail if none + const changedWidgets = getWidgetsWithChangedPositions(this.dashboard.widgets); + if (!changedWidgets.length) { + this.isLayoutDirty = false; + $scope.$applyAsync(); + return; + } + this.saveInProgress = true; - const showMessages = true; return $q - .all(_.map(widgets, widget => widget.save())) + .all(_.map(changedWidgets, widget => widget.save())) .then(() => { - if (showMessages) { - notification.success('Changes saved.'); - } - // Update original widgets positions - _.each(widgets, (widget) => { - _.extend(widget.$originalPosition, widget.options.position); - }); + this.isLayoutDirty = false; }) .catch(() => { - if (showMessages) { - notification.error('Error saving changes.'); - } + // in the off-chance that a widget got deleted mid-saving it's position, an error will occur + // currently left unhandled PR 3653#issuecomment-481699053 + notification.error('Error saving changes.'); }) .finally(() => { this.saveInProgress = false; }); }; + const saveDashboardLayoutDebounced = _.debounce(saveDashboardLayout, 2000); + this.layoutEditing = false; this.isFullscreen = false; this.refreshRate = null; @@ -84,6 +87,7 @@ function DashboardCtrl( this.showPermissionsControl = clientConfig.showPermissionsControl; this.globalParameters = []; this.isDashboardOwner = false; + this.isLayoutDirty = false; this.refreshRates = clientConfig.dashboardRefreshIntervals.map(interval => ({ name: durationHumanize(interval), @@ -242,28 +246,17 @@ function DashboardCtrl( }); }; - this.editLayout = (enableEditing, applyChanges) => { - if (!this.isGridDisabled) { - if (!enableEditing) { - if (applyChanges) { - const changedWidgets = getWidgetsWithChangedPositions(this.dashboard.widgets); - saveDashboardLayout(changedWidgets); - } else { - // Revert changes - const items = {}; - _.each(this.dashboard.widgets, (widget) => { - _.extend(widget.options.position, widget.$originalPosition); - items[widget.id] = widget.options.position; - }); - this.dashboard.widgets = Dashboard.prepareWidgetsForDashboard(this.dashboard.widgets); - if (this.updateGridItems) { - this.updateGridItems(items); - } - } - } - - this.layoutEditing = enableEditing; + this.onLayoutChanged = () => { + // prevent unnecessary save when gridstack is loaded + if (!this.layoutEditing) { + return; } + this.isLayoutDirty = true; + saveDashboardLayoutDebounced(); + }; + + this.editLayout = (enableEditing) => { + this.layoutEditing = enableEditing; }; this.loadTags = () => getTags('api/dashboards/tags').then(tags => _.map(tags, t => t.name)); @@ -405,12 +398,11 @@ function DashboardCtrl( this.removeWidget = (widgetId) => { this.dashboard.widgets = this.dashboard.widgets.filter(w => w.id !== undefined && w.id !== widgetId); this.extractGlobalParameters(); + $scope.$applyAsync(); + if (!this.layoutEditing) { // We need to wait a bit while `angular` updates widgets, and only then save new layout - $timeout(() => { - const changedWidgets = getWidgetsWithChangedPositions(this.dashboard.widgets); - saveDashboardLayout(changedWidgets); - }, 50); + $timeout(saveDashboardLayout, 50); } }; diff --git a/client/app/pages/dashboards/dashboard.less b/client/app/pages/dashboards/dashboard.less index f3ddf23770..a2a469c917 100644 --- a/client/app/pages/dashboards/dashboard.less +++ b/client/app/pages/dashboards/dashboard.less @@ -112,6 +112,38 @@ } } +.dashboard__control { + .save-status { + vertical-align: middle; + margin-right: 7px; + font-size: 12px; + text-align: left; + display: inline-block; + + &[data-saving] { + opacity: 0.6; + width: 45px; + + &:after { + content: ''; + animation: saving 2s linear infinite; + } + } + } +} + +@keyframes saving { + 0%, 100% { + content: '.'; + } + 33% { + content: '..'; + } + 66% { + content: '...'; + } +} + // Mobile fixes @media (max-width: 767px) { diff --git a/client/app/services/widget.js b/client/app/services/widget.js index 9bd4221e96..36540500c1 100644 --- a/client/app/services/widget.js +++ b/client/app/services/widget.js @@ -92,6 +92,10 @@ function WidgetFactory($http, $location, Query, Visualization, dashboardGridOpti this.options.position.autoHeight = true; } + this.updateOriginalPosition(); + } + + updateOriginalPosition() { // Save original position (create a shallow copy) this.$originalPosition = extend({}, this.options.position); } @@ -161,6 +165,8 @@ function WidgetFactory($http, $location, Query, Visualization, dashboardGridOpti this[k] = v; }); + this.updateOriginalPosition(); + return this; }); } diff --git a/client/cypress/integration/dashboard/dashboard_spec.js b/client/cypress/integration/dashboard/dashboard_spec.js index c6e05fdbca..1e0db5f2c9 100644 --- a/client/cypress/integration/dashboard/dashboard_spec.js +++ b/client/cypress/integration/dashboard/dashboard_spec.js @@ -360,46 +360,14 @@ describe('Dashboard', () => { }); }); - it('discards drag on cancel', () => { - let start; - cy.get('@textboxEl') - // save initial position, drag textbox 1 col - .then(($el) => { - start = $el.offset(); - editDashboard(); - return dragBy(cy.get('@textboxEl'), 200); - }) - // cancel - .then(() => { - cy.get('.dashboard-header').within(() => { - cy.contains('button', 'Cancel').click(); - }); - return cy.get('@textboxEl'); - }) - // verify returned to original position - .then(($el) => { - expect($el.offset()).to.deep.eq(start); - }); - }); + it('auto saves after drag', () => { + cy.server(); + cy.route('POST', 'api/widgets/*').as('WidgetSave'); - it('saves drag on apply', () => { - let start; - cy.get('@textboxEl') - // save initial position, drag textbox 1 col - .then(($el) => { - start = $el.offset(); - editDashboard(); - return dragBy(cy.get('@textboxEl'), 200); - }) - // apply - .then(() => { - cy.contains('button', 'Apply Changes').click(); - return cy.get('@textboxEl'); - }) - // verify move - .then(($el) => { - expect($el.offset()).to.not.deep.eq(start); - }); + editDashboard(); + dragBy(cy.get('@textboxEl'), 330).then(() => { + cy.wait('@WidgetSave'); + }); }); }); @@ -458,46 +426,14 @@ describe('Dashboard', () => { }); }); - it('discards resize on cancel', () => { - let start; - cy.get('@textboxEl') - // save initial position, resize textbox 1 col - .then(($el) => { - start = $el.height(); - editDashboard(); - return resizeBy(cy.get('@textboxEl'), 0, 200); - }) - // cancel - .then(() => { - cy.get('.dashboard-header').within(() => { - cy.contains('button', 'Cancel').click(); - }); - return cy.get('@textboxEl'); - }) - // verify returned to original size - .then(($el) => { - expect($el.height()).to.eq(start); - }); - }); + it('auto saves after resize', () => { + cy.server(); + cy.route('POST', 'api/widgets/*').as('WidgetSave'); - it('saves resize on apply', () => { - let start; - cy.get('@textboxEl') - // save initial position, resize textbox 1 col - .then(($el) => { - start = $el.height(); - editDashboard(); - return resizeBy(cy.get('@textboxEl'), 0, 200); - }) - // apply - .then(() => { - cy.contains('button', 'Apply Changes').click().should('not.exist'); - return cy.get('@textboxEl'); - }) - // verify size change persists - .then(($el) => { - expect($el.height()).to.not.eq(start); - }); + editDashboard(); + resizeBy(cy.get('@textboxEl'), 200).then(() => { + cy.wait('@WidgetSave'); + }); }); }); }); @@ -678,7 +614,7 @@ describe('Dashboard', () => { it('disables edit mode', function () { cy.visit(this.dashboardEditUrl); - cy.contains('button', 'Apply Changes') + cy.contains('button', 'Done Editing') .as('saveButton') .should('be.disabled'); From 97492d7aa0f0eded339adf960ffc08db60796807 Mon Sep 17 00:00:00 2001 From: Arik Fraimovich Date: Wed, 17 Apr 2019 11:13:45 +0300 Subject: [PATCH 054/179] Fix: update default CSP policy to allow KB iframe. (#3714) ## What type of PR is this? (check all applicable) - [x] Bug Fix ## Description Without this change the Help Drawer couldn't load content anymore. ## Related Tickets & Documents #3404 --- redash/settings/__init__.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/redash/settings/__init__.py b/redash/settings/__init__.py index 53e96aacd0..2bb024d8b6 100644 --- a/redash/settings/__init__.py +++ b/redash/settings/__init__.py @@ -17,6 +17,7 @@ def all_settings(): return settings + REDIS_URL = os.environ.get('REDASH_REDIS_URL', os.environ.get('REDIS_URL', "redis://localhost:6379/0")) PROXIES_COUNT = int(os.environ.get('REDASH_PROXIES_COUNT', "1")) @@ -110,7 +111,7 @@ def all_settings(): # for more information. E.g.: CONTENT_SECURITY_POLICY = os.environ.get( "REDASH_CONTENT_SECURITY_POLICY", - "default-src 'self'; style-src 'self' 'unsafe-inline'; script-src 'self' 'unsafe-eval'; font-src 'self' data:; img-src 'self' http: https: data:; object-src 'none'; frame-ancestors 'none';" + "default-src 'self'; style-src 'self' 'unsafe-inline'; script-src 'self' 'unsafe-eval'; font-src 'self' data:; img-src 'self' http: https: data:; object-src 'none'; frame-ancestors 'none'; frame-src redash.io;" ) CONTENT_SECURITY_POLICY_REPORT_URI = os.environ.get( "REDASH_CONTENT_SECURITY_POLICY_REPORT_URI", "") From aa9d2466cdfc1972390a80b7c4bba2121e49f11c Mon Sep 17 00:00:00 2001 From: Jannis Leidel Date: Thu, 18 Apr 2019 18:39:38 +0200 Subject: [PATCH 055/179] Split redash/__init__.py to prevent import time side-effects. (#3601) ## What type of PR is this? (check all applicable) - [x] Refactor - [x] Bug Fix ## Description This basically makes sure that when import the redash package we don't accidentally trigger import-time side-effects such as requiring Redis. Refs #3569 and #3466. --- redash/__init__.py | 87 +++++-------------------------------- redash/app.py | 56 ++++++++++++++++++++++++ redash/cli/__init__.py | 10 +++-- redash/handlers/base.py | 2 +- redash/settings/__init__.py | 14 +----- redash/tasks/queries.py | 2 +- redash/worker.py | 4 +- tests/__init__.py | 4 +- 8 files changed, 82 insertions(+), 97 deletions(-) create mode 100644 redash/app.py diff --git a/redash/__init__.py b/redash/__init__.py index cc50844c89..1abf9c3413 100644 --- a/redash/__init__.py +++ b/redash/__init__.py @@ -1,27 +1,22 @@ -import sys import logging -import urlparse +import os +import sys import urllib +import urlparse import redis -from flask import Flask, current_app -from werkzeug.contrib.fixers import ProxyFix -from werkzeug.routing import BaseConverter -from statsd import StatsClient from flask_mail import Mail from flask_limiter import Limiter from flask_limiter.util import get_ipaddr from flask_migrate import Migrate +from statsd import StatsClient -from redash import settings -from redash.query_runner import import_query_runners -from redash.destinations import import_destinations - +from . import settings +from .app import create_app # noqa __version__ = '7.0.0' -import os if os.environ.get("REMOTE_DEBUG"): import ptvsd ptvsd.enable_attach(address=('0.0.0.0', 5678)) @@ -36,10 +31,8 @@ def setup_logging(): # Make noisy libraries less noisy if settings.LOG_LEVEL != "DEBUG": - logging.getLogger("passlib").setLevel("ERROR") - logging.getLogger("requests.packages.urllib3").setLevel("ERROR") - logging.getLogger("snowflake.connector").setLevel("ERROR") - logging.getLogger('apiclient').setLevel("ERROR") + for name in ["passlib", "requests.packages.urllib3", "snowflake.connector", "apiclient"]: + logging.getLogger(name).setLevel("ERROR") def create_redis_connection(): @@ -67,69 +60,13 @@ def create_redis_connection(): setup_logging() + redis_connection = create_redis_connection() mail = Mail() -migrate = Migrate() -mail.init_mail(settings.all_settings()) -statsd_client = StatsClient(host=settings.STATSD_HOST, port=settings.STATSD_PORT, prefix=settings.STATSD_PREFIX) -limiter = Limiter(key_func=get_ipaddr, storage_uri=settings.LIMITER_STORAGE) - -import_query_runners(settings.QUERY_RUNNERS) -import_destinations(settings.DESTINATIONS) -from redash.version_check import reset_new_version_status -reset_new_version_status() - - -class SlugConverter(BaseConverter): - def to_python(self, value): - # This is ay workaround for when we enable multi-org and some files are being called by the index rule: - # for path in settings.STATIC_ASSETS_PATHS: - # full_path = safe_join(path, value) - # if os.path.isfile(full_path): - # raise ValidationError() - - return value - - def to_url(self, value): - return value - - -def create_app(): - from redash import authentication, extensions, handlers, security - from redash.handlers.webpack import configure_webpack - from redash.handlers import chrome_logger - from redash.models import db, users - from redash.metrics import request as request_metrics - from redash.utils import sentry - - sentry.init() - - app = Flask(__name__, - template_folder=settings.STATIC_ASSETS_PATH, - static_folder=settings.STATIC_ASSETS_PATH, - static_url_path='/static') - - # Make sure we get the right referral address even behind proxies like nginx. - app.wsgi_app = ProxyFix(app.wsgi_app, settings.PROXIES_COUNT) - app.url_map.converters['org_slug'] = SlugConverter - - # configure our database - app.config['SQLALCHEMY_DATABASE_URI'] = settings.SQLALCHEMY_DATABASE_URI - app.config.update(settings.all_settings()) +migrate = Migrate() - security.init_app(app) - request_metrics.init_app(app) - db.init_app(app) - migrate.init_app(app, db) - mail.init_app(app) - authentication.init_app(app) - limiter.init_app(app) - handlers.init_app(app) - configure_webpack(app) - extensions.init_app(app) - chrome_logger.init_app(app) - users.init_app(app) +statsd_client = StatsClient(host=settings.STATSD_HOST, port=settings.STATSD_PORT, prefix=settings.STATSD_PREFIX) - return app +limiter = Limiter(key_func=get_ipaddr, storage_uri=settings.LIMITER_STORAGE) diff --git a/redash/app.py b/redash/app.py new file mode 100644 index 0000000000..ea9e3d5536 --- /dev/null +++ b/redash/app.py @@ -0,0 +1,56 @@ +from flask import Flask +from werkzeug.contrib.fixers import ProxyFix + +from . import settings + + +class Redash(Flask): + """A custom Flask app for Redash""" + def __init__(self, *args, **kwargs): + kwargs.update({ + 'template_folder': settings.STATIC_ASSETS_PATH, + 'static_folder': settings.STATIC_ASSETS_PATH, + 'static_path': '/static', + }) + super(Redash, self).__init__(__name__, *args, **kwargs) + # Make sure we get the right referral address even behind proxies like nginx. + self.wsgi_app = ProxyFix(self.wsgi_app, settings.PROXIES_COUNT) + # Configure Redash using our settings + self.config.from_object('redash.settings') + + +def create_app(): + from . import authentication, extensions, handlers, limiter, mail, migrate, security + from .destinations import import_destinations + from .handlers import chrome_logger + from .handlers.webpack import configure_webpack + from .metrics import request as request_metrics + from .models import db, users + from .query_runner import import_query_runners + from .utils import sentry + from .version_check import reset_new_version_status + + sentry.init() + app = Redash() + + # Check and update the cached version for use by the client + app.before_first_request(reset_new_version_status) + + # Load query runners and destinations + import_query_runners(settings.QUERY_RUNNERS) + import_destinations(settings.DESTINATIONS) + + security.init_app(app) + request_metrics.init_app(app) + db.init_app(app) + migrate.init_app(app, db) + mail.init_app(app) + authentication.init_app(app) + limiter.init_app(app) + handlers.init_app(app) + configure_webpack(app) + extensions.init_app(app) + chrome_logger.init_app(app) + users.init_app(app) + + return app diff --git a/redash/cli/__init__.py b/redash/cli/__init__.py index 316e426d5d..fdf714ad70 100644 --- a/redash/cli/__init__.py +++ b/redash/cli/__init__.py @@ -15,9 +15,11 @@ def create(group): @app.shell_context_processor def shell_context(): - from redash import models - return dict(models=models) - + from redash import models, settings + return { + 'models': models, + 'settings': settings, + } return app @@ -48,7 +50,7 @@ def status(): @manager.command() def check_settings(): """Show the settings as Redash sees them (useful for debugging).""" - for name, item in settings.all_settings().iteritems(): + for name, item in current_app.config.iteritems(): print("{} = {}".format(name, item)) diff --git a/redash/handlers/base.py b/redash/handlers/base.py index 05e82d6c2a..e4bc8e66b2 100644 --- a/redash/handlers/base.py +++ b/redash/handlers/base.py @@ -117,7 +117,7 @@ def paginate(query_set, page, page_size, serializer, **kwargs): def org_scoped_rule(rule): if settings.MULTI_ORG: - return "/{}".format(rule) + return "/{}".format(rule) return rule diff --git a/redash/settings/__init__.py b/redash/settings/__init__.py index 2bb024d8b6..1b260177d4 100644 --- a/redash/settings/__init__.py +++ b/redash/settings/__init__.py @@ -4,19 +4,7 @@ from flask_talisman import talisman from .helpers import fix_assets_path, array_from_string, parse_boolean, int_or_none, set_from_string -from .organization import DATE_FORMAT - - -def all_settings(): - from types import ModuleType - - settings = {} - for name, item in globals().iteritems(): - if not callable(item) and not name.startswith("__") and not isinstance(item, ModuleType): - settings[name] = item - - return settings - +from .organization import DATE_FORMAT # noqa REDIS_URL = os.environ.get('REDASH_REDIS_URL', os.environ.get('REDIS_URL', "redis://localhost:6379/0")) PROXIES_COUNT = int(os.environ.get('REDASH_PROXIES_COUNT', "1")) diff --git a/redash/tasks/queries.py b/redash/tasks/queries.py index ab16209138..6002ccd275 100644 --- a/redash/tasks/queries.py +++ b/redash/tasks/queries.py @@ -11,7 +11,7 @@ from redash import models, redis_connection, settings, statsd_client from redash.query_runner import InterruptException from redash.tasks.alerts import check_alerts_for_query -from redash.utils import gen_query_hash, json_dumps, json_loads, utcnow, mustache_render +from redash.utils import gen_query_hash, json_dumps, utcnow, mustache_render from redash.worker import celery logger = get_task_logger(__name__) diff --git a/redash/worker.py b/redash/worker.py index 23c3d7b6fe..b46db432e2 100644 --- a/redash/worker.py +++ b/redash/worker.py @@ -8,8 +8,10 @@ from celery import Celery from celery.schedules import crontab from celery.signals import worker_process_init + from redash import create_app, settings -from redash.metrics import celery as celery_metrics +from redash.metrics import celery as celery_metrics # noqa + celery = Celery('redash', broker=settings.CELERY_BROKER, diff --git a/tests/__init__.py b/tests/__init__.py index 07a8fe8f85..9d0f0832b2 100644 --- a/tests/__init__.py +++ b/tests/__init__.py @@ -16,8 +16,8 @@ # Make sure rate limit is enabled os.environ['REDASH_RATELIMIT_ENABLED'] = "true" -from redash import create_app, limiter -from redash import redis_connection +from redash import limiter, redis_connection +from redash.app import create_app from redash.models import db from redash.utils import json_dumps, json_loads from tests.factories import Factory, user_factory From fea082ec7794ae98569b012179e106cfc4706c79 Mon Sep 17 00:00:00 2001 From: Gabriel Dutra Date: Fri, 19 Apr 2019 10:58:37 -0300 Subject: [PATCH 056/179] Update Percy network idle timeout (#3724) --- client/cypress/cypress.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/client/cypress/cypress.js b/client/cypress/cypress.js index 3d27ba86da..1410228b5c 100644 --- a/client/cypress/cypress.js +++ b/client/cypress/cypress.js @@ -53,7 +53,7 @@ function runCypressCI() { } execSync( - 'docker-compose run cypress ./node_modules/.bin/percy exec -- ./node_modules/.bin/cypress run --record', + 'docker-compose run cypress ./node_modules/.bin/percy exec -t 300 -- ./node_modules/.bin/cypress run --record', { stdio: 'inherit' }, ); } From 10a6ccbbcdc0f553d5a3448950a2b9265f10eb96 Mon Sep 17 00:00:00 2001 From: Ran Byron Date: Fri, 19 Apr 2019 21:41:35 +0300 Subject: [PATCH 057/179] Dashboard save fail indication (#3715) --- .../assets/less/redash/redash-newstyle.less | 4 --- client/app/pages/dashboards/dashboard.html | 33 +++++++++++++------ client/app/pages/dashboards/dashboard.js | 19 +++++++++-- client/app/pages/dashboards/dashboard.less | 4 +++ 4 files changed, 43 insertions(+), 17 deletions(-) diff --git a/client/app/assets/less/redash/redash-newstyle.less b/client/app/assets/less/redash/redash-newstyle.less index 8808113960..28a0389568 100644 --- a/client/app/assets/less/redash/redash-newstyle.less +++ b/client/app/assets/less/redash/redash-newstyle.less @@ -937,8 +937,4 @@ text.slicetext { .markdown strong { font-weight: bold; -} - -.disabled-silent { - pointer-events: none; } \ No newline at end of file diff --git a/client/app/pages/dashboards/dashboard.html b/client/app/pages/dashboards/dashboard.html index cc987db29b..46f3912e7c 100644 --- a/client/app/pages/dashboards/dashboard.html +++ b/client/app/pages/dashboards/dashboard.html @@ -15,16 +15,29 @@

-
- Saving - Saved - - +
+ + + Saving + + + + Saving Failed + + + + + Saved + +
-
+
diff --git a/client/app/pages/dashboards/dashboard.html b/client/app/pages/dashboards/dashboard.html index 46f3912e7c..35559987ea 100644 --- a/client/app/pages/dashboards/dashboard.html +++ b/client/app/pages/dashboards/dashboard.html @@ -101,10 +101,9 @@

-
+
+ is-one-column-mode="$ctrl.isGridDisabled" class="dashboard-wrapper">
diff --git a/client/app/pages/dashboards/dashboard.less b/client/app/pages/dashboards/dashboard.less index 42a429e6bd..2dda95f1de 100644 --- a/client/app/pages/dashboards/dashboard.less +++ b/client/app/pages/dashboards/dashboard.less @@ -17,7 +17,7 @@ overflow: visible; } - &.preview-mode { + .preview-mode & { .widget-menu-regular { display: block; } @@ -26,7 +26,7 @@ } } - &.editing-mode { + .editing-mode & { .widget-menu-regular { display: none; } @@ -176,9 +176,52 @@ public-dashboard-page { .container { min-height: calc(100vh - 95px); } - + #footer { height: 95px; text-align: center; } } + +/**** + grid bg - based on 6 cols, 35px rows and 15px spacing +****/ + +// let the bg go all the way to the bottom +dashboard-page, dashboard-page .container { + display: flex; + flex-grow: 1; + flex-direction: column; + width: 100%; +} + +#dashboard-container { + position: relative; + flex-grow: 1; + margin-bottom: 50px; // but not ALL the way ಠ_ಠ + + &.editing-mode.grid-enabled { + /* Y axis lines */ + background: linear-gradient(to right, transparent, transparent 1px, #F6F8F9 1px, #F6F8F9), linear-gradient(to bottom, #B3BABF, #B3BABF 1px, transparent 1px, transparent); + background-size: 5px 50px; + background-position-y: -8px; + + /* X axis lines */ + &::before { + content: ""; + position: absolute; + top: 0; + left: 0; + bottom: 0; + right: 15px; + background: linear-gradient(to bottom, transparent, transparent 2px, #F6F8F9 2px, #F6F8F9 5px), linear-gradient(to left, #B3BABF, #B3BABF 1px, transparent 1px, transparent); + background-size: calc((100vw - 15px) / 6) 5px; + background-position: -7px 1px; + } + } +} + +// placeholder bg color +.grid-stack-placeholder > .placeholder-content { + background-color: rgba(224, 230, 235, 0.5) !important; +} From 48955b5fa1cb40232ca40cd28b68066823b27026 Mon Sep 17 00:00:00 2001 From: Osmo Salomaa Date: Mon, 29 Apr 2019 21:21:51 +0300 Subject: [PATCH 064/179] Use monospace font in query output log (#3743) Closes #3739 --- client/app/assets/less/redash/query.less | 6 ++++++ client/app/pages/queries/query.html | 2 +- 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/client/app/assets/less/redash/query.less b/client/app/assets/less/redash/query.less index 157c8b6823..8b58e80c15 100644 --- a/client/app/assets/less/redash/query.less +++ b/client/app/assets/less/redash/query.less @@ -172,6 +172,12 @@ edit-in-place p.editable:hover { } } +.query-log-line { + font-family: monospace; + white-space: pre; + margin: 0; +} + .paginator-container { text-align: center; } diff --git a/client/app/pages/queries/query.html b/client/app/pages/queries/query.html index 1333e10ed1..842b319550 100644 --- a/client/app/pages/queries/query.html +++ b/client/app/pages/queries/query.html @@ -224,7 +224,7 @@

Log Information:

-

{{l}}

+

{{l}}

    From c9bf4122401aa9c5c7444493fed08167c9f48897 Mon Sep 17 00:00:00 2001 From: guwenqing <43440728+guwenqing@users.noreply.github.com> Date: Mon, 29 Apr 2019 20:23:06 +0200 Subject: [PATCH 065/179] Update npm run to fix hpe_header_overflow (#3732) Nodejs has set max header size to 8k in http_parser, need to provide a larger header size to make the proxy work. --- package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package.json b/package.json index 2f0fb57556..0428f1a97e 100644 --- a/package.json +++ b/package.json @@ -4,7 +4,7 @@ "description": "The frontend part of Redash.", "main": "index.js", "scripts": { - "start": "webpack-dev-server", + "start": "node --max-http-header-size=16385 ./node_modules/webpack-dev-server/bin/webpack-dev-server.js", "bundle": "bin/bundle-extensions", "clean": "rm -rf ./client/dist/", "build": "npm run clean && NODE_ENV=production node --max-old-space-size=4096 node_modules/.bin/webpack", From f3a653c57f61adbfd7ae8194e61236aac8d1cff1 Mon Sep 17 00:00:00 2001 From: Gabriel Dutra Date: Mon, 29 Apr 2019 15:50:04 -0300 Subject: [PATCH 066/179] Fix query based parameter has value null when created (#3707) * Fix query based parameter value null when created * Use toString to avoid having 'null' string --- client/app/components/QueryBasedParameterInput.jsx | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/client/app/components/QueryBasedParameterInput.jsx b/client/app/components/QueryBasedParameterInput.jsx index e2e731cd03..7273a34619 100644 --- a/client/app/components/QueryBasedParameterInput.jsx +++ b/client/app/components/QueryBasedParameterInput.jsx @@ -1,4 +1,4 @@ -import { find, isFunction } from 'lodash'; +import { find, isFunction, toString } from 'lodash'; import React from 'react'; import PropTypes from 'prop-types'; import { react2angular } from 'react2angular'; @@ -35,10 +35,9 @@ export class QueryBasedParameterInput extends React.Component { this._loadOptions(this.props.queryId); } - // eslint-disable-next-line no-unused-vars - componentWillReceiveProps(nextProps) { - if (nextProps.queryId !== this.props.queryId) { - this._loadOptions(nextProps.queryId, nextProps.value); + componentDidUpdate(prevProps) { + if (this.props.queryId !== prevProps.queryId) { + this._loadOptions(this.props.queryId); } } @@ -68,7 +67,7 @@ export class QueryBasedParameterInput extends React.Component { className={className} disabled={loading || (options.length === 0)} loading={loading} - defaultValue={'' + value} + value={toString(value)} onChange={onSelect} dropdownMatchSelectWidth={false} dropdownClassName="ant-dropdown-in-bootstrap-modal" From 21e22a2d0d4375d46d61ccb401d82059673fff0c Mon Sep 17 00:00:00 2001 From: Omer Lachish Date: Mon, 29 Apr 2019 21:58:29 +0300 Subject: [PATCH 067/179] add get_by_id to Organization (#3712) --- redash/models/organizations.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/redash/models/organizations.py b/redash/models/organizations.py index fcf8cc8c2f..28df95b918 100644 --- a/redash/models/organizations.py +++ b/redash/models/organizations.py @@ -32,6 +32,10 @@ def __str__(self): def get_by_slug(cls, slug): return cls.query.filter(cls.slug == slug).first() + @classmethod + def get_by_id(cls, _id): + return cls.query.filter(cls.id == _id).one() + @property def default_group(self): return self.groups.filter(Group.name == 'default', Group.type == Group.BUILTIN_GROUP).first() From d0b2151b4d89e3f8b3c2583a564cb44fd5111052 Mon Sep 17 00:00:00 2001 From: Ran Byron Date: Mon, 29 Apr 2019 23:29:17 +0300 Subject: [PATCH 068/179] Fix query page height (#3744) --- client/app/assets/less/redash/query.less | 1 + 1 file changed, 1 insertion(+) diff --git a/client/app/assets/less/redash/query.less b/client/app/assets/less/redash/query.less index 8b58e80c15..7c250c9fff 100644 --- a/client/app/assets/less/redash/query.less +++ b/client/app/assets/less/redash/query.less @@ -264,6 +264,7 @@ a.label-tag { .query-page-wrapper { display: flex; flex-direction: column; + flex-grow: 1; } .query-fullscreen { From 9a4433bf685eec1f3fb24639ee11ab3e9fd8b115 Mon Sep 17 00:00:00 2001 From: Levko Kravets Date: Tue, 30 Apr 2019 16:34:00 +0300 Subject: [PATCH 069/179] Migrate visualizations registry/renderer/editor to React (#3493) --- client/app/assets/less/ant.less | 68 ++- .../assets/less/inc/visualizations/misc.less | 2 + .../less/inc/visualizations/pivot-table.less | 2 +- client/app/assets/less/redash/query.less | 4 +- client/app/components/ColorBox.jsx | 23 + client/app/components/Filters.jsx | 122 ++++ client/app/components/color-box.less | 8 + client/app/components/dashboards/widget.html | 6 +- client/app/components/dashboards/widget.js | 1 + client/app/components/filters.html | 44 -- client/app/components/filters.js | 30 - client/app/lib/hooks/useQueryResult.js | 30 + client/app/lib/utils.js | 21 +- client/app/lib/visualizations/sunburst.js | 15 +- client/app/pages/dashboards/dashboard.html | 7 +- client/app/pages/dashboards/dashboard.js | 40 +- client/app/pages/dashboards/dashboard.less | 10 +- .../dashboards/public-dashboard-page.html | 4 +- .../pages/dashboards/public-dashboard-page.js | 8 +- client/app/pages/queries/query.html | 21 +- client/app/pages/queries/view.js | 66 +-- client/app/services/dashboard.js | 30 + client/app/services/query-result.js | 177 +----- client/app/services/query.js | 5 - client/app/services/visualization.js | 9 + client/app/services/widget.js | 31 +- client/app/visualizations/ColorPalette.js | 38 ++ .../EditVisualizationDialog.jsx | 213 +++++++ .../app/visualizations/VisualizationName.jsx | 23 + .../visualizations/VisualizationRenderer.jsx | 76 +++ client/app/visualizations/box-plot/Editor.jsx | 39 ++ .../box-plot/box-plot-editor.html | 15 - client/app/visualizations/box-plot/index.js | 346 ++++++----- .../visualizations/chart/chart-editor.html | 235 ++++---- client/app/visualizations/chart/chart.html | 6 +- .../app/visualizations/chart/getChartData.js | 101 ++++ client/app/visualizations/chart/index.js | 556 ++++++++---------- .../app/visualizations/chart/plotly/index.js | 2 - .../app/visualizations/chart/plotly/utils.js | 31 +- .../choropleth/choropleth-editor.html | 102 ++-- .../visualizations/choropleth/choropleth.html | 10 +- client/app/visualizations/choropleth/index.js | 504 ++++++++-------- .../visualizations/cohort/cohort-editor.html | 32 +- client/app/visualizations/cohort/index.js | 161 +++-- .../counter/counter-editor.html | 38 +- client/app/visualizations/counter/index.js | 244 ++++---- .../edit-visualization-dialog.html | 43 -- .../edit-visualization-dialog.js | 99 ---- .../visualizations/funnel/funnel-editor.html | 17 +- client/app/visualizations/funnel/index.js | 138 +++-- client/app/visualizations/index.js | 220 +++---- client/app/visualizations/map/index.js | 505 ++++++++-------- client/app/visualizations/map/map-editor.html | 38 +- client/app/visualizations/pivot/Editor.jsx | 26 + client/app/visualizations/pivot/index.js | 138 ++--- client/app/visualizations/pivot/pivot.less | 10 +- .../pivot/pivottable-editor.html | 2 +- client/app/visualizations/sankey/Editor.jsx | 20 + client/app/visualizations/sankey/index.js | 106 ++-- .../visualizations/sankey/sankey-editor.html | 14 - client/app/visualizations/sunburst/Editor.jsx | 33 ++ client/app/visualizations/sunburst/index.js | 78 +-- .../sunburst/sunburst-sequence-editor.html | 23 - client/app/visualizations/table/index.js | 163 +++-- .../visualizations/table/table-editor.html | 22 +- client/app/visualizations/table/table.html | 2 +- .../app/visualizations/word-cloud/Editor.jsx | 31 + client/app/visualizations/word-cloud/index.js | 192 +++--- .../word-cloud/word-cloud-editor.html | 8 - .../integration/dashboard/dashboard_spec.js | 128 ++-- .../edit_visualization_dialog_spec.js | 29 + .../integration/visualizations/pivot_spec.js | 69 +++ .../visualizations/sankey_sunburst_spec.js | 51 ++ client/cypress/support/redash-api/index.js | 70 +++ package-lock.json | 22 + package.json | 1 + 76 files changed, 3170 insertions(+), 2684 deletions(-) create mode 100644 client/app/components/ColorBox.jsx create mode 100644 client/app/components/Filters.jsx create mode 100644 client/app/components/color-box.less delete mode 100644 client/app/components/filters.html delete mode 100644 client/app/components/filters.js create mode 100644 client/app/lib/hooks/useQueryResult.js create mode 100644 client/app/services/visualization.js create mode 100644 client/app/visualizations/ColorPalette.js create mode 100644 client/app/visualizations/EditVisualizationDialog.jsx create mode 100644 client/app/visualizations/VisualizationName.jsx create mode 100644 client/app/visualizations/VisualizationRenderer.jsx create mode 100644 client/app/visualizations/box-plot/Editor.jsx delete mode 100644 client/app/visualizations/box-plot/box-plot-editor.html create mode 100644 client/app/visualizations/chart/getChartData.js delete mode 100644 client/app/visualizations/edit-visualization-dialog.html delete mode 100644 client/app/visualizations/edit-visualization-dialog.js create mode 100644 client/app/visualizations/pivot/Editor.jsx create mode 100644 client/app/visualizations/sankey/Editor.jsx delete mode 100644 client/app/visualizations/sankey/sankey-editor.html create mode 100644 client/app/visualizations/sunburst/Editor.jsx delete mode 100644 client/app/visualizations/sunburst/sunburst-sequence-editor.html create mode 100644 client/app/visualizations/word-cloud/Editor.jsx delete mode 100644 client/app/visualizations/word-cloud/word-cloud-editor.html create mode 100644 client/cypress/integration/visualizations/edit_visualization_dialog_spec.js create mode 100644 client/cypress/integration/visualizations/pivot_spec.js create mode 100644 client/cypress/integration/visualizations/sankey_sunburst_spec.js create mode 100644 client/cypress/support/redash-api/index.js diff --git a/client/app/assets/less/ant.less b/client/app/assets/less/ant.less index f05dfcd2bd..d4993f9c86 100644 --- a/client/app/assets/less/ant.less +++ b/client/app/assets/less/ant.less @@ -221,21 +221,61 @@ } } -// styling for short modals (no lines) -.@{dialog-prefix-cls}.shortModal { - .@{dialog-prefix-cls} { - &-header, - &-footer { - border: none; - padding: 16px; - } - &-body { - padding: 10px 16px; +.@{dialog-prefix-cls} { + // styling for short modals (no lines) + &.shortModal { + .@{dialog-prefix-cls} { + &-header, + &-footer { + border: none; + padding: 16px; + } + + &-body { + padding: 10px 16px; + } + + &-close-x { + width: 46px; + height: 46px; + line-height: 46px; + } } - &-close-x { - width: 46px; - height: 46px; - line-height: 46px; + } + + // fullscreen modals + &-fullscreen { + .@{dialog-prefix-cls} { + position: absolute; + left: 15px; + top: 15px; + right: 15px; + bottom: 15px; + width: auto !important; + height: auto !important; + max-width: none; + max-height: none; + margin: 0; + padding: 0; + + .@{dialog-prefix-cls}-content { + position: absolute; + left: 0; + top: 0; + right: 0; + bottom: 0; + width: auto; + height: auto; + margin: 0; + padding: 0; + display: flex; + flex-direction: column; + } + + .@{dialog-prefix-cls}-body { + flex: 1 1 auto; + overflow: auto; + } } } } diff --git a/client/app/assets/less/inc/visualizations/misc.less b/client/app/assets/less/inc/visualizations/misc.less index d439837db0..cc5600bd16 100644 --- a/client/app/assets/less/inc/visualizations/misc.less +++ b/client/app/assets/less/inc/visualizations/misc.less @@ -1,4 +1,6 @@ visualization-renderer { + display: block; + .pagination, .ant-pagination { margin: 0; diff --git a/client/app/assets/less/inc/visualizations/pivot-table.less b/client/app/assets/less/inc/visualizations/pivot-table.less index 9fa85ae5ca..7400914f47 100644 --- a/client/app/assets/less/inc/visualizations/pivot-table.less +++ b/client/app/assets/less/inc/visualizations/pivot-table.less @@ -1,3 +1,3 @@ -pivot-table-renderer > table, grid-renderer > div, visualization-renderer > div { +.pivot-table-renderer > table, grid-renderer > div, visualization-renderer > div { overflow: auto; } diff --git a/client/app/assets/less/redash/query.less b/client/app/assets/less/redash/query.less index 7c250c9fff..6cf1b8d3e2 100644 --- a/client/app/assets/less/redash/query.less +++ b/client/app/assets/less/redash/query.less @@ -222,7 +222,7 @@ edit-in-place p.editable:hover { .widget-wrapper { .body-container { - filters { + .filters-wrapper { display: block; padding-left: 15px; } @@ -343,7 +343,7 @@ a.label-tag { border-bottom: 1px solid #efefef; } - pivot-table-renderer > table, grid-renderer > div, visualization-renderer > div { + .pivot-table-renderer > table, grid-renderer > div, visualization-renderer > div { overflow: visible; } diff --git a/client/app/components/ColorBox.jsx b/client/app/components/ColorBox.jsx new file mode 100644 index 0000000000..73b3f3681e --- /dev/null +++ b/client/app/components/ColorBox.jsx @@ -0,0 +1,23 @@ +import React from 'react'; +import PropTypes from 'prop-types'; +import { react2angular } from 'react2angular'; + +import './color-box.less'; + +export function ColorBox({ color }) { + return ; +} + +ColorBox.propTypes = { + color: PropTypes.string, +}; + +ColorBox.defaultProps = { + color: 'transparent', +}; + +export default function init(ngModule) { + ngModule.component('colorBox', react2angular(ColorBox)); +} + +init.init = true; diff --git a/client/app/components/Filters.jsx b/client/app/components/Filters.jsx new file mode 100644 index 0000000000..f9da7f56ed --- /dev/null +++ b/client/app/components/Filters.jsx @@ -0,0 +1,122 @@ +import { isArray, map, includes, every, some } from 'lodash'; +import moment from 'moment'; +import React from 'react'; +import PropTypes from 'prop-types'; +import { react2angular } from 'react2angular'; +import Select from 'antd/lib/select'; + +const ALL_VALUES = '###Redash::Filters::SelectAll###'; +const NONE_VALUES = '###Redash::Filters::Clear###'; + +export const FilterType = PropTypes.shape({ + name: PropTypes.string.isRequired, + friendlyName: PropTypes.string.isRequired, + multiple: PropTypes.bool, + current: PropTypes.oneOfType([ + PropTypes.any, + PropTypes.arrayOf(PropTypes.any), + ]).isRequired, + values: PropTypes.arrayOf(PropTypes.any).isRequired, +}); + +export const FiltersType = PropTypes.arrayOf(FilterType); + +function createFilterChangeHandler(filters, onChange) { + return (filter, value) => { + if (filter.multiple && includes(value, ALL_VALUES)) { + value = [...filter.values]; + } + if (filter.multiple && includes(value, NONE_VALUES)) { + value = []; + } + filters = map(filters, f => (f.name === filter.name ? { ...filter, current: value } : f)); + onChange(filters); + }; +} + +export function filterData(rows, filters = []) { + if (!isArray(rows)) { + return []; + } + + let result = rows; + + if (isArray(filters) && (filters.length > 0)) { + // "every" field's value should match "some" of corresponding filter's values + result = result.filter(row => every( + filters, + (filter) => { + const rowValue = row[filter.name]; + const filterValues = isArray(filter.current) ? filter.current : [filter.current]; + return some(filterValues, (filterValue) => { + if (moment.isMoment(rowValue)) { + return rowValue.isSame(filterValue); + } + // We compare with either the value or the String representation of the value, + // because Select2 casts true/false to "true"/"false". + return (filterValue === rowValue) || (String(rowValue) === filterValue); + }); + }, + )); + } + + return result; +} + +export function Filters({ filters, onChange }) { + if (filters.length === 0) { + return null; + } + + onChange = createFilterChangeHandler(filters, onChange); + + return ( +
    +
    +
    + {map(filters, (filter) => { + const options = map(filter.values, value => ( + {value} + )); + + return ( +
    + + +
    + ); + })} +
    +
    +
    + ); +} + +Filters.propTypes = { + filters: FiltersType.isRequired, + onChange: PropTypes.func, // (name, value) => void +}; + +Filters.defaultProps = { + onChange: () => {}, +}; + +export default function init(ngModule) { + ngModule.component('filters', react2angular(Filters)); +} + +init.init = true; diff --git a/client/app/components/color-box.less b/client/app/components/color-box.less new file mode 100644 index 0000000000..e1027258a4 --- /dev/null +++ b/client/app/components/color-box.less @@ -0,0 +1,8 @@ +color-box { + span { + width: 12px !important; + height: 12px !important; + display: inline-block !important; + margin-right: 5px; + } +} diff --git a/client/app/components/dashboards/widget.html b/client/app/components/dashboards/widget.html index 9e6765174a..3b08aded89 100644 --- a/client/app/components/dashboards/widget.html +++ b/client/app/components/dashboards/widget.html @@ -46,7 +46,11 @@
    Error running query: {{$ctrl.widget.getQueryResult().getError()}}
- +
diff --git a/client/app/pages/dashboards/public-dashboard-page.js b/client/app/pages/dashboards/public-dashboard-page.js index 7cac64741d..5b769168e2 100644 --- a/client/app/pages/dashboards/public-dashboard-page.js +++ b/client/app/pages/dashboards/public-dashboard-page.js @@ -12,7 +12,7 @@ const PublicDashboardPage = { bindings: { dashboard: '<', }, - controller($timeout, $location, $http, $route, dashboardGridOptions, Dashboard) { + controller($scope, $timeout, $location, $http, $route, dashboardGridOptions, Dashboard) { 'ngInject'; this.dashboardGridOptions = Object.assign({}, dashboardGridOptions, { @@ -32,6 +32,12 @@ const PublicDashboardPage = { this.dashboard = data; this.dashboard.widgets = Dashboard.prepareDashboardWidgets(this.dashboard.widgets); + this.filters = []; // TODO: implement (@/services/dashboard.js:collectDashboardFilters) + this.filtersOnChange = (allFilters) => { + this.filters = allFilters; + $scope.$applyAsync(); + }; + $timeout(refresh, refreshRate * 1000.0); }); }; diff --git a/client/app/pages/queries/query.html b/client/app/pages/queries/query.html index 842b319550..a482c9ed20 100644 --- a/client/app/pages/queries/query.html +++ b/client/app/pages/queries/query.html @@ -226,25 +226,20 @@

Log Information:

{{l}}

-
@@ -255,7 +250,11 @@

- +
+ ng-click="$ctrl.options.legend.alignText = 'left'" + ng-class="{active: $ctrl.options.legend.alignText == 'left'}"> + ng-click="$ctrl.options.legend.alignText = 'center'" + ng-class="{active: $ctrl.options.legend.alignText == 'center'}"> + ng-click="$ctrl.options.legend.alignText = 'right'" + ng-class="{active: $ctrl.options.legend.alignText == 'right'}">

- +
+ ng-model="$ctrl.options.tooltip.template" ng-model-options="{ allowInvalid: true, debounce: 200 }" + ng-disabled="!$ctrl.options.tooltip.enabled">
- +
+ ng-model="$ctrl.options.popup.template" ng-model-options="{ allowInvalid: true, debounce: 200 }" + ng-disabled="!$ctrl.options.popup.enabled">
-
+
+ ng-model="$ctrl.options.steps">
- +
@@ -138,12 +138,12 @@
- + - + @@ -154,12 +154,12 @@
- + - + @@ -170,12 +170,12 @@
- + - + @@ -188,12 +188,12 @@
- + - + @@ -204,12 +204,12 @@
- + - + @@ -219,17 +219,17 @@
-
+
+ ng-model="$ctrl.options.bounds[1][0]" ng-model-options="{ allowInvalid: true, debounce: 200 }">
+ ng-model="$ctrl.options.bounds[1][1]" ng-model-options="{ allowInvalid: true, debounce: 200 }">
@@ -239,11 +239,11 @@
+ ng-model="$ctrl.options.bounds[0][0]" ng-model-options="{ allowInvalid: true, debounce: 200 }">
+ ng-model="$ctrl.options.bounds[0][1]" ng-model-options="{ allowInvalid: true, debounce: 200 }">
diff --git a/client/app/visualizations/choropleth/choropleth.html b/client/app/visualizations/choropleth/choropleth.html index 232a884b7f..bd1c8093cd 100644 --- a/client/app/visualizations/choropleth/choropleth.html +++ b/client/app/visualizations/choropleth/choropleth.html @@ -1,11 +1,11 @@
-
-
+
-
+
-
{{ formatValue(item.limit) }}
+
{{ $ctrl.formatValue(item.limit) }}
diff --git a/client/app/visualizations/choropleth/index.js b/client/app/visualizations/choropleth/index.js index ff45d47e59..d5dd238178 100644 --- a/client/app/visualizations/choropleth/index.js +++ b/client/app/visualizations/choropleth/index.js @@ -4,6 +4,9 @@ import 'leaflet/dist/leaflet.css'; import { formatSimpleTemplate } from '@/lib/value-format'; import 'leaflet-fullscreen'; import 'leaflet-fullscreen/dist/leaflet.fullscreen.css'; +import { angular2react } from 'angular2react'; +import { registerVisualization } from '@/visualizations'; +import ColorPalette from '@/visualizations/ColorPalette'; import { AdditionalColors, @@ -22,6 +25,38 @@ import editorTemplate from './choropleth-editor.html'; import countriesDataUrl from './countries.geo.json'; +export const ChoroplethPalette = _.extend({}, AdditionalColors, ColorPalette); + +const DEFAULT_OPTIONS = { + countryCodeColumn: '', + countryCodeType: 'iso_a3', + valueColumn: '', + clusteringMode: 'e', + steps: 5, + valueFormat: '0,0.00', + noValuePlaceholder: 'N/A', + colors: { + min: ChoroplethPalette['Light Blue'], + max: ChoroplethPalette['Dark Blue'], + background: ChoroplethPalette.White, + borders: ChoroplethPalette.White, + noValue: ChoroplethPalette['Light Gray'], + }, + legend: { + visible: true, + position: 'bottom-left', + alignText: 'right', + }, + tooltip: { + enabled: true, + template: '{{ @@name }}: {{ @@value }}', + }, + popup: { + enabled: true, + template: 'Country: {{ @@name_long }} ({{ @@iso_a2 }})\n
\nValue: {{ @@value }}', + }, +}; + const loadCountriesData = _.bind(function loadCountriesData($http, url) { if (!this[url]) { this[url] = $http.get(url).then(response => response.data); @@ -29,283 +64,248 @@ const loadCountriesData = _.bind(function loadCountriesData($http, url) { return this[url]; }, {}); -function choroplethRenderer($sanitize, $http) { - return { - restrict: 'E', - template, - scope: { - queryResult: '=', - options: '=?', - }, - link($scope, $element) { - let countriesData = null; - let map = null; - let choropleth = null; - let updateBoundsLock = false; - - function getBounds() { - if (!updateBoundsLock) { - const bounds = map.getBounds(); - $scope.options.bounds = [ - [bounds._southWest.lat, bounds._southWest.lng], - [bounds._northEast.lat, bounds._northEast.lng], - ]; - $scope.$applyAsync(); - } +const ChoroplethRenderer = { + template, + bindings: { + data: '<', + options: '<', + onOptionsChange: '<', + }, + controller($scope, $element, $sanitize, $http) { + let countriesData = null; + let map = null; + let choropleth = null; + let mapMoveLock = false; + + const onMapMoveStart = () => { + mapMoveLock = true; + }; + + const onMapMoveEnd = () => { + const bounds = map.getBounds(); + this.options.bounds = [ + [bounds._southWest.lat, bounds._southWest.lng], + [bounds._northEast.lat, bounds._northEast.lng], + ]; + if (this.onOptionsChange) { + this.onOptionsChange(this.options); } + $scope.$applyAsync(() => { + mapMoveLock = false; + }); + }; - function setBounds({ disableAnimation = false } = {}) { - if (map && choropleth) { - const bounds = $scope.options.bounds || choropleth.getBounds(); - const options = disableAnimation ? { - animate: false, - duration: 0, - } : null; - map.fitBounds(bounds, options); - } + const updateBounds = ({ disableAnimation = false } = {}) => { + if (mapMoveLock) { + return; } + if (map && choropleth) { + const bounds = this.options.bounds || choropleth.getBounds(); + const options = disableAnimation ? { + animate: false, + duration: 0, + } : null; + map.fitBounds(bounds, options); + } + }; - function render() { - if (map) { - map.remove(); - map = null; - choropleth = null; - } - if (!countriesData) { - return; - } - - $scope.formatValue = createNumberFormatter( - $scope.options.valueFormat, - $scope.options.noValuePlaceholder, - ); - - const data = prepareData( - $scope.queryResult.getData(), - $scope.options.countryCodeColumn, - $scope.options.valueColumn, - ); - - const { limits, colors, legend } = createScale( - countriesData.features, - data, - $scope.options, - ); - - // Update data for legend block - $scope.legendItems = legend; - - choropleth = L.geoJson(countriesData, { - onEachFeature: (feature, layer) => { - const value = getValueForFeature(feature, data, $scope.options.countryCodeType); - const valueFormatted = $scope.formatValue(value); - const featureData = prepareFeatureProperties( - feature, - valueFormatted, - data, - $scope.options.countryCodeType, - ); - const color = getColorByValue(value, limits, colors, $scope.options.colors.noValue); + const render = () => { + if (map) { + map.remove(); + map = null; + choropleth = null; + } + if (!countriesData) { + return; + } + this.formatValue = createNumberFormatter( + this.options.valueFormat, + this.options.noValuePlaceholder, + ); + + const data = prepareData(this.data.rows, this.options.countryCodeColumn, this.options.valueColumn); + + const { limits, colors, legend } = createScale(countriesData.features, data, this.options); + + // Update data for legend block + this.legendItems = legend; + + choropleth = L.geoJson(countriesData, { + onEachFeature: (feature, layer) => { + const value = getValueForFeature(feature, data, this.options.countryCodeType); + const valueFormatted = this.formatValue(value); + const featureData = prepareFeatureProperties( + feature, + valueFormatted, + data, + this.options.countryCodeType, + ); + const color = getColorByValue(value, limits, colors, this.options.colors.noValue); + + layer.setStyle({ + color: this.options.colors.borders, + weight: 1, + fillColor: color, + fillOpacity: 1, + }); + + if (this.options.tooltip.enabled) { + layer.bindTooltip($sanitize(formatSimpleTemplate( + this.options.tooltip.template, + featureData, + ))); + } + + if (this.options.popup.enabled) { + layer.bindPopup($sanitize(formatSimpleTemplate( + this.options.popup.template, + featureData, + ))); + } + + layer.on('mouseover', () => { + layer.setStyle({ + weight: 2, + fillColor: darkenColor(color), + }); + }); + layer.on('mouseout', () => { layer.setStyle({ - color: $scope.options.colors.borders, weight: 1, fillColor: color, - fillOpacity: 1, }); + }); + }, + }); - if ($scope.options.tooltip.enabled) { - layer.bindTooltip($sanitize(formatSimpleTemplate( - $scope.options.tooltip.template, - featureData, - ))); - } - - if ($scope.options.popup.enabled) { - layer.bindPopup($sanitize(formatSimpleTemplate( - $scope.options.popup.template, - featureData, - ))); - } - - layer.on('mouseover', () => { - layer.setStyle({ - weight: 2, - fillColor: darkenColor(color), - }); - }); - layer.on('mouseout', () => { - layer.setStyle({ - weight: 1, - fillColor: color, - }); - }); - }, - }); - - const choroplethBounds = choropleth.getBounds(); - - map = L.map($element[0].children[0].children[0], { - center: choroplethBounds.getCenter(), - zoom: 1, - zoomSnap: 0, - layers: [choropleth], - scrollWheelZoom: false, - maxBounds: choroplethBounds, - maxBoundsViscosity: 1, - attributionControl: false, - fullscreenControl: true, - }); - - map.on('focus', () => { map.on('moveend', getBounds); }); - map.on('blur', () => { map.off('moveend', getBounds); }); - - setBounds({ disableAnimation: true }); - } + const choroplethBounds = choropleth.getBounds(); + + map = L.map($element[0].children[0].children[0], { + center: choroplethBounds.getCenter(), + zoom: 1, + zoomSnap: 0, + layers: [choropleth], + scrollWheelZoom: false, + maxBounds: choroplethBounds, + maxBoundsViscosity: 1, + attributionControl: false, + fullscreenControl: true, + }); - loadCountriesData($http, countriesDataUrl).then((data) => { - if (_.isObject(data)) { - countriesData = data; - render(); - } + map.on('focus', () => { + map.on('movestart', onMapMoveStart); + map.on('moveend', onMapMoveEnd); + }); + map.on('blur', () => { + map.off('movestart', onMapMoveStart); + map.off('moveend', onMapMoveEnd); }); - $scope.handleResize = _.debounce(() => { - if (map) { - map.invalidateSize(false); - setBounds({ disableAnimation: true }); - } - }, 50); - - $scope.$watch('queryResult && queryResult.getData()', render); - $scope.$watch(() => _.omit($scope.options, 'bounds'), render, true); - $scope.$watch('options.bounds', () => { - // Prevent infinite digest loop - const savedLock = updateBoundsLock; - updateBoundsLock = true; - setBounds(); - updateBoundsLock = savedLock; - }, true); - }, - }; -} + updateBounds({ disableAnimation: true }); + }; -function choroplethEditor(ChoroplethPalette) { - return { - restrict: 'E', - template: editorTemplate, - scope: { - queryResult: '=', - options: '=?', - }, - link($scope) { - $scope.currentTab = 'general'; - $scope.changeTab = (tab) => { - $scope.currentTab = tab; - }; - - $scope.colors = ChoroplethPalette; - - $scope.clusteringModes = { - q: 'quantile', - e: 'equidistant', - k: 'k-means', - }; - - $scope.legendPositions = { - 'top-left': 'top / left', - 'top-right': 'top / right', - 'bottom-left': 'bottom / left', - 'bottom-right': 'bottom / right', - }; - - $scope.countryCodeTypes = { - name: 'Short name', - name_long: 'Full name', - abbrev: 'Abbreviated name', - iso_a2: 'ISO code (2 letters)', - iso_a3: 'ISO code (3 letters)', - iso_n3: 'ISO code (3 digits)', - }; - - $scope.templateHint = ` -
All query result columns can be referenced using {{ column_name }} syntax.
-
Use special names to access additional properties:
-
{{ @@value }} formatted value;
-
{{ @@name }} short country name;
-
{{ @@name_long }} full country name;
-
{{ @@abbrev }} abbreviated country name;
-
{{ @@iso_a2 }} two-letter ISO country code;
-
{{ @@iso_a3 }} three-letter ISO country code;
-
{{ @@iso_n3 }} three-digit ISO country code.
-
This syntax is applicable to tooltip and popup templates.
- `; - - function updateCountryCodeType() { - $scope.options.countryCodeType = inferCountryCodeType( - $scope.queryResult.getData(), - $scope.options.countryCodeColumn, - ) || $scope.options.countryCodeType; + loadCountriesData($http, countriesDataUrl).then((data) => { + if (_.isObject(data)) { + countriesData = data; + render(); } + }); - $scope.$watch('options.countryCodeColumn', updateCountryCodeType); - $scope.$watch('queryResult.getData()', updateCountryCodeType); - }, - }; -} + $scope.handleResize = _.debounce(() => { + if (map) { + map.invalidateSize(false); + updateBounds({ disableAnimation: true }); + } + }, 50); + + $scope.$watch('$ctrl.data', render); + $scope.$watch(() => _.omit(this.options, 'bounds'), render, true); + $scope.$watch('$ctrl.options.bounds', updateBounds, true); + }, +}; + +const ChoroplethEditor = { + template: editorTemplate, + bindings: { + data: '<', + options: '<', + onOptionsChange: '<', + }, + controller($scope) { + this.currentTab = 'general'; + this.setCurrentTab = (tab) => { + this.currentTab = tab; + }; -export default function init(ngModule) { - ngModule.constant('ChoroplethPalette', {}); - ngModule.directive('choroplethRenderer', choroplethRenderer); - ngModule.directive('choroplethEditor', choroplethEditor); - ngModule.config((VisualizationProvider, ColorPalette, ChoroplethPalette) => { - _.extend(ChoroplethPalette, AdditionalColors, ColorPalette); + this.colors = ChoroplethPalette; - const renderTemplate = - ''; + this.clusteringModes = { + q: 'quantile', + e: 'equidistant', + k: 'k-means', + }; - const editTemplate = ''; + this.legendPositions = { + 'top-left': 'top / left', + 'top-right': 'top / right', + 'bottom-left': 'bottom / left', + 'bottom-right': 'bottom / right', + }; - const defaultOptions = { - defaultColumns: 3, - defaultRows: 8, - minColumns: 2, + this.countryCodeTypes = { + name: 'Short name', + name_long: 'Full name', + abbrev: 'Abbreviated name', + iso_a2: 'ISO code (2 letters)', + iso_a3: 'ISO code (3 letters)', + iso_n3: 'ISO code (3 digits)', + }; - countryCodeColumn: '', - countryCodeType: 'iso_a3', - valueColumn: '', - clusteringMode: 'e', - steps: 5, - valueFormat: '0,0.00', - noValuePlaceholder: 'N/A', - colors: { - min: ChoroplethPalette['Light Blue'], - max: ChoroplethPalette['Dark Blue'], - background: ChoroplethPalette.White, - borders: ChoroplethPalette.White, - noValue: ChoroplethPalette['Light Gray'], - }, - legend: { - visible: true, - position: 'bottom-left', - alignText: 'right', - }, - tooltip: { - enabled: true, - template: '{{ @@name }}: {{ @@value }}', - }, - popup: { - enabled: true, - template: 'Country: {{ @@name_long }} ({{ @@iso_a2 }})\n
\nValue: {{ @@value }}', - }, + this.templateHint = ` +
All query result columns can be referenced using {{ column_name }} syntax.
+
Use special names to access additional properties:
+
{{ @@value }} formatted value;
+
{{ @@name }} short country name;
+
{{ @@name_long }} full country name;
+
{{ @@abbrev }} abbreviated country name;
+
{{ @@iso_a2 }} two-letter ISO country code;
+
{{ @@iso_a3 }} three-letter ISO country code;
+
{{ @@iso_n3 }} three-digit ISO country code.
+
This syntax is applicable to tooltip and popup templates.
+ `; + + const updateCountryCodeType = () => { + this.options.countryCodeType = inferCountryCodeType( + this.data ? this.data.rows : [], + this.options.countryCodeColumn, + ) || this.options.countryCodeType; }; - VisualizationProvider.registerVisualization({ + $scope.$watch('$ctrl.options.countryCodeColumn', updateCountryCodeType); + $scope.$watch('$ctrl.data', updateCountryCodeType); + + $scope.$watch('$ctrl.options', (options) => { + this.onOptionsChange(options); + }, true); + }, +}; + +export default function init(ngModule) { + ngModule.component('choroplethRenderer', ChoroplethRenderer); + ngModule.component('choroplethEditor', ChoroplethEditor); + + ngModule.run(($injector) => { + registerVisualization({ type: 'CHOROPLETH', name: 'Map (Choropleth)', - renderTemplate, - editorTemplate: editTemplate, - defaultOptions, + getOptions: options => _.merge({}, DEFAULT_OPTIONS, options), + Renderer: angular2react('choroplethRenderer', ChoroplethRenderer, $injector), + Editor: angular2react('choroplethEditor', ChoroplethEditor, $injector), + + defaultColumns: 3, + defaultRows: 8, + minColumns: 2, }); }); } diff --git a/client/app/visualizations/cohort/cohort-editor.html b/client/app/visualizations/cohort/cohort-editor.html index 33b60699ea..9e49b75ec2 100644 --- a/client/app/visualizations/cohort/cohort-editor.html +++ b/client/app/visualizations/cohort/cohort-editor.html @@ -1,16 +1,16 @@ -
+
- @@ -19,35 +19,35 @@
-
-
+
- +
- +
- +
- +
diff --git a/client/app/visualizations/cohort/index.js b/client/app/visualizations/cohort/index.js index 3125820b96..764517c0d1 100644 --- a/client/app/visualizations/cohort/index.js +++ b/client/app/visualizations/cohort/index.js @@ -3,6 +3,8 @@ import _ from 'lodash'; import moment from 'moment'; import 'cornelius/src/cornelius'; import 'cornelius/src/cornelius.css'; +import { angular2react } from 'angular2react'; +import { registerVisualization } from '@/visualizations'; import editorTemplate from './cohort-editor.html'; @@ -19,9 +21,6 @@ const DEFAULT_OPTIONS = { stageColumn: 'day_number', totalColumn: 'total', valueColumn: 'value', - - autoHeight: true, - defaultRows: 8, }; function groupData(sortedData) { @@ -135,103 +134,89 @@ function prepareData(rawData, options) { return { data, initialDate }; } -function cohortRenderer() { - return { - restrict: 'E', - scope: { - queryResult: '=', - options: '=', - }, - template: '', - replace: false, - link($scope, element) { - $scope.options = _.extend({}, DEFAULT_OPTIONS, $scope.options); - - function updateCohort() { - element.empty(); - - if ($scope.queryResult.getData() === null) { - return; - } +const CohortRenderer = { + bindings: { + data: '<', + options: '<', + }, + template: '', + replace: false, + controller($scope, $element) { + $scope.options = _.extend({}, DEFAULT_OPTIONS, $scope.options); + + const update = () => { + $element.empty(); + + if (this.data.rows.length === 0) { + return; + } - const columnNames = _.map($scope.queryResult.getColumns(), i => i.name); - if ( - !_.includes(columnNames, $scope.options.dateColumn) || - !_.includes(columnNames, $scope.options.stageColumn) || - !_.includes(columnNames, $scope.options.totalColumn) || - !_.includes(columnNames, $scope.options.valueColumn) - ) { - return; - } + const options = this.options; - const { data, initialDate } = prepareData($scope.queryResult.getData(), $scope.options); - - Cornelius.draw({ - initialDate, - container: element[0], - cohort: data, - title: null, - timeInterval: $scope.options.timeInterval, - labels: { - time: 'Time', - people: 'Users', - weekOf: 'Week of', - }, - }); + const columnNames = _.map(this.data.columns, i => i.name); + if ( + !_.includes(columnNames, options.dateColumn) || + !_.includes(columnNames, options.stageColumn) || + !_.includes(columnNames, options.totalColumn) || + !_.includes(columnNames, options.valueColumn) + ) { + return; } - $scope.$watch('queryResult && queryResult.getData()', updateCohort); - $scope.$watch('options', updateCohort, true); - }, - }; -} + const { data, initialDate } = prepareData(this.data.rows, options); + + Cornelius.draw({ + initialDate, + container: $element[0], + cohort: data, + title: null, + timeInterval: options.timeInterval, + labels: { + time: 'Time', + people: 'Users', + weekOf: 'Week of', + }, + }); + }; -function cohortEditor() { - return { - restrict: 'E', - template: editorTemplate, - link: ($scope) => { - $scope.visualization.options = _.extend({}, DEFAULT_OPTIONS, $scope.visualization.options); - - $scope.currentTab = 'columns'; - $scope.setCurrentTab = (tab) => { - $scope.currentTab = tab; - }; - - function refreshColumns() { - $scope.columns = $scope.queryResult.getColumns(); - $scope.columnNames = _.map($scope.columns, i => i.name); - } + $scope.$watch('$ctrl.data', update); + $scope.$watch('$ctrl.options', update, true); + }, +}; - refreshColumns(); +const CohortEditor = { + template: editorTemplate, + bindings: { + data: '<', + options: '<', + onOptionsChange: '<', + }, + controller($scope) { + this.currentTab = 'columns'; + this.setCurrentTab = (tab) => { + this.currentTab = tab; + }; - $scope.$watch( - () => [$scope.queryResult.getId(), $scope.queryResult.status], - (changed) => { - if (!changed[0] || changed[1] !== 'done') { - return; - } - refreshColumns(); - }, - true, - ); - }, - }; -} + $scope.$watch('$ctrl.options', (options) => { + this.onOptionsChange(options); + }, true); + }, +}; export default function init(ngModule) { - ngModule.directive('cohortRenderer', cohortRenderer); - ngModule.directive('cohortEditor', cohortEditor); - - ngModule.config((VisualizationProvider) => { - const editTemplate = ''; + ngModule.component('cohortRenderer', CohortRenderer); + ngModule.component('cohortEditor', CohortEditor); - VisualizationProvider.registerVisualization({ + ngModule.run(($injector) => { + registerVisualization({ type: 'COHORT', name: 'Cohort', - renderTemplate: '', - editorTemplate: editTemplate, - defaultOptions: DEFAULT_OPTIONS, + getOptions: options => ({ ...DEFAULT_OPTIONS, ...options }), + Renderer: angular2react('cohortRenderer', CohortRenderer, $injector), + Editor: angular2react('cohortEditor', CohortEditor, $injector), + + autoHeight: true, + defaultRows: 8, }); }); } diff --git a/client/app/visualizations/counter/counter-editor.html b/client/app/visualizations/counter/counter-editor.html index d6f5cb75de..82ecd199a6 100644 --- a/client/app/visualizations/counter/counter-editor.html +++ b/client/app/visualizations/counter/counter-editor.html @@ -1,84 +1,86 @@
-
+
- +
- +
- +
-
-
+
- +
-
+
- +
- +
- +
- +
- +
diff --git a/client/app/visualizations/counter/index.js b/client/app/visualizations/counter/index.js index 15fa98e107..b930576f0d 100644 --- a/client/app/visualizations/counter/index.js +++ b/client/app/visualizations/counter/index.js @@ -1,9 +1,22 @@ import numberFormat from 'underscore.string/numberFormat'; import { isNumber } from 'lodash'; +import { angular2react } from 'angular2react'; +import { registerVisualization } from '@/visualizations'; import counterTemplate from './counter.html'; import counterEditorTemplate from './counter-editor.html'; +const DEFAULT_OPTIONS = { + counterLabel: '', + counterColName: 'counter', + rowNumber: 1, + targetRowNumber: 1, + stringDecimal: 0, + stringDecChar: '.', + stringThouSep: ',', +}; + +// TODO: Need to review this function, it does not properly handle edge cases. function getRowNumber(index, size) { if (index >= 0) { return index - 1; @@ -16,134 +29,137 @@ function getRowNumber(index, size) { return size + index; } -function CounterRenderer($timeout) { - return { - restrict: 'E', - template: counterTemplate, - link($scope, $element) { - $scope.fontSize = '1em'; - - $scope.scale = 1; - const root = $element[0].querySelector('counter'); - const container = $element[0].querySelector('counter > div'); - $scope.handleResize = () => { - const scale = Math.min(root.offsetWidth / container.offsetWidth, root.offsetHeight / container.offsetHeight); - $scope.scale = Math.floor(scale * 100) / 100; // keep only two decimal places - }; - - const refreshData = () => { - const queryData = $scope.queryResult.getData(); - if (queryData) { - const rowNumber = getRowNumber($scope.visualization.options.rowNumber, queryData.length); - const targetRowNumber = getRowNumber($scope.visualization.options.targetRowNumber, queryData.length); - const counterColName = $scope.visualization.options.counterColName; - const targetColName = $scope.visualization.options.targetColName; - const counterLabel = $scope.visualization.options.counterLabel; - - if (counterLabel) { - $scope.counterLabel = counterLabel; - } else { - $scope.counterLabel = $scope.visualization.name; - } +const CounterRenderer = { + template: counterTemplate, + bindings: { + data: '<', + options: '<', + visualizationName: '<', + }, + controller($scope, $element, $timeout) { + $scope.fontSize = '1em'; + + $scope.scale = 1; + const root = $element[0].querySelector('counter'); + const container = $element[0].querySelector('counter > div'); + $scope.handleResize = () => { + const scale = Math.min(root.offsetWidth / container.offsetWidth, root.offsetHeight / container.offsetHeight); + $scope.scale = Math.floor(scale * 100) / 100; // keep only two decimal places + }; - if ($scope.visualization.options.countRow) { - $scope.counterValue = queryData.length; - } else if (counterColName) { - $scope.counterValue = queryData[rowNumber][counterColName]; - } - if (targetColName) { - $scope.targetValue = queryData[targetRowNumber][targetColName]; - - if ($scope.targetValue) { - $scope.delta = $scope.counterValue - $scope.targetValue; - $scope.trendPositive = $scope.delta >= 0; - } - } else { - $scope.targetValue = null; + const update = () => { + const options = this.options; + const data = this.data.rows; + + if (data.length > 0) { + const rowNumber = getRowNumber(options.rowNumber, data.length); + const targetRowNumber = getRowNumber(options.targetRowNumber, data.length); + const counterColName = options.counterColName; + const targetColName = options.targetColName; + const counterLabel = options.counterLabel; + + if (counterLabel) { + $scope.counterLabel = counterLabel; + } else { + $scope.counterLabel = this.visualizationName; + } + + if (options.countRow) { + $scope.counterValue = data.length; + } else if (counterColName) { + $scope.counterValue = data[rowNumber][counterColName]; + } + if (targetColName) { + $scope.targetValue = data[targetRowNumber][targetColName]; + + if ($scope.targetValue) { + $scope.delta = $scope.counterValue - $scope.targetValue; + $scope.trendPositive = $scope.delta >= 0; } + } else { + $scope.targetValue = null; + } - $scope.isNumber = isNumber($scope.counterValue); - if ($scope.isNumber) { - $scope.stringPrefix = $scope.visualization.options.stringPrefix; - $scope.stringSuffix = $scope.visualization.options.stringSuffix; - - const stringDecimal = $scope.visualization.options.stringDecimal; - const stringDecChar = $scope.visualization.options.stringDecChar; - const stringThouSep = $scope.visualization.options.stringThouSep; - if (stringDecimal || stringDecChar || stringThouSep) { - $scope.counterValue = numberFormat($scope.counterValue, stringDecimal, stringDecChar, stringThouSep); - $scope.isNumber = false; - } - } else { - $scope.stringPrefix = null; - $scope.stringSuffix = null; + $scope.isNumber = isNumber($scope.counterValue); + if ($scope.isNumber) { + $scope.stringPrefix = options.stringPrefix; + $scope.stringSuffix = options.stringSuffix; + + const stringDecimal = options.stringDecimal; + const stringDecChar = options.stringDecChar; + const stringThouSep = options.stringThouSep; + if (stringDecimal || stringDecChar || stringThouSep) { + $scope.counterValue = numberFormat($scope.counterValue, stringDecimal, stringDecChar, stringThouSep); + $scope.isNumber = false; } + } else { + $scope.stringPrefix = null; + $scope.stringSuffix = null; } + } - $timeout(() => { - $scope.handleResize(); - }); - }; + $timeout(() => { + $scope.handleResize(); + }); + }; - $scope.$watch('visualization.options', refreshData, true); - $scope.$watch('queryResult && queryResult.getData()', refreshData); - }, - }; -} + $scope.$watch('$ctrl.data', update); + $scope.$watch('$ctrl.options', update, true); + }, +}; + +const CounterEditor = { + template: counterEditorTemplate, + bindings: { + data: '<', + options: '<', + visualizationName: '<', + onOptionsChange: '<', + }, + controller($scope) { + this.currentTab = 'general'; + this.changeTab = (tab) => { + this.currentTab = tab; + }; -function CounterEditor() { - return { - restrict: 'E', - template: counterEditorTemplate, - link(scope) { - scope.currentTab = 'general'; - scope.changeTab = (tab) => { - scope.currentTab = tab; - }; - scope.isValueNumber = () => { - const queryData = scope.queryResult.getData(); - if (queryData) { - const rowNumber = getRowNumber(scope.visualization.options.rowNumber, queryData.length); - const counterColName = scope.visualization.options.counterColName; - - if (scope.visualization.options.countRow) { - scope.counterValue = queryData.length; - } else if (counterColName) { - scope.counterValue = queryData[rowNumber][counterColName]; - } + this.isValueNumber = () => { + const options = this.options; + const data = this.data.rows; + + if (data.length > 0) { + const rowNumber = getRowNumber(options.rowNumber, data.length); + const counterColName = options.counterColName; + + if (options.countRow) { + this.counterValue = data.length; + } else if (counterColName) { + this.counterValue = data[rowNumber][counterColName]; } - return isNumber(scope.counterValue); - }; - }, - }; -} + } -export default function init(ngModule) { - ngModule.directive('counterEditor', CounterEditor); - ngModule.directive('counterRenderer', CounterRenderer); - - ngModule.config((VisualizationProvider) => { - const renderTemplate = - ''; - - const editTemplate = ''; - const defaultOptions = { - counterColName: 'counter', - rowNumber: 1, - targetRowNumber: 1, - stringDecimal: 0, - stringDecChar: '.', - stringThouSep: ',', - defaultColumns: 2, - defaultRows: 5, + return isNumber(this.counterValue); }; - VisualizationProvider.registerVisualization({ + $scope.$watch('$ctrl.options', (options) => { + this.onOptionsChange(options); + }, true); + }, +}; + +export default function init(ngModule) { + ngModule.component('counterRenderer', CounterRenderer); + ngModule.component('counterEditor', CounterEditor); + + ngModule.run(($injector) => { + registerVisualization({ type: 'COUNTER', name: 'Counter', - renderTemplate, - editorTemplate: editTemplate, - defaultOptions, + getOptions: options => ({ ...DEFAULT_OPTIONS, ...options }), + Renderer: angular2react('counterRenderer', CounterRenderer, $injector), + Editor: angular2react('counterEditor', CounterEditor, $injector), + + defaultColumns: 2, + defaultRows: 5, }); }); } diff --git a/client/app/visualizations/edit-visualization-dialog.html b/client/app/visualizations/edit-visualization-dialog.html deleted file mode 100644 index 28791ee2ca..0000000000 --- a/client/app/visualizations/edit-visualization-dialog.html +++ /dev/null @@ -1,43 +0,0 @@ -
-
- -

Visualization Editor

-
-
-
-
-
- -
- - - -
- -
- - -
- - - - -
-
-
- -
-
-
-
- - -
-
diff --git a/client/app/visualizations/edit-visualization-dialog.js b/client/app/visualizations/edit-visualization-dialog.js deleted file mode 100644 index 918d8600fd..0000000000 --- a/client/app/visualizations/edit-visualization-dialog.js +++ /dev/null @@ -1,99 +0,0 @@ -import { map } from 'lodash'; -import { copy } from 'angular'; -import notification from '@/services/notification'; -import template from './edit-visualization-dialog.html'; - -const EditVisualizationDialog = { - template, - bindings: { - resolve: '<', - close: '&', - dismiss: '&', - }, - controller($window, currentUser, Events, Visualization) { - 'ngInject'; - - this.query = this.resolve.query; - this.queryResult = this.resolve.queryResult; - this.originalVisualization = this.resolve.visualization; - this.onNewSuccess = this.resolve.onNewSuccess; - this.visualization = copy(this.originalVisualization); - this.visTypes = Visualization.visualizationTypes; - - // Don't allow to change type after creating visualization - this.canChangeType = !(this.visualization && this.visualization.id); - - this.newVisualization = () => ({ - type: Visualization.defaultVisualization.type, - name: Visualization.defaultVisualization.name, - description: '', - options: Visualization.defaultVisualization.defaultOptions, - }); - if (!this.visualization) { - this.visualization = this.newVisualization(); - } - - this.typeChanged = (oldType) => { - const type = this.visualization.type; - // if not edited by user, set name to match type - // todo: this is wrong, because he might have edited it before. - if (type && oldType !== type && this.visualization && !this.visForm.name.$dirty) { - this.visualization.name = Visualization.visualizations[this.visualization.type].name; - } - - // Bring default options - if (type && oldType !== type && this.visualization) { - this.visualization.options = Visualization.visualizations[this.visualization.type].defaultOptions; - } - }; - - this.submit = () => { - if (this.visualization.id) { - Events.record('update', 'visualization', this.visualization.id, { type: this.visualization.type }); - } else { - Events.record('create', 'visualization', null, { type: this.visualization.type }); - } - - this.visualization.query_id = this.query.id; - - Visualization.save( - this.visualization, - (result) => { - notification.success('Visualization saved'); - - const visIds = map(this.query.visualizations, i => i.id); - const index = visIds.indexOf(result.id); - if (index > -1) { - this.query.visualizations[index] = result; - } else { - // new visualization - this.query.visualizations.push(result); - if (this.onNewSuccess) { - this.onNewSuccess(result); - } - } - this.close(); - }, - () => { - notification.error('Visualization could not be saved'); - }, - ); - }; - - this.closeDialog = () => { - if (this.visForm.$dirty) { - if ($window.confirm('Are you sure you want to close the editor without saving?')) { - this.close(); - } - } else { - this.close(); - } - }; - }, -}; - -export default function init(ngModule) { - ngModule.component('editVisualizationDialog', EditVisualizationDialog); -} - -init.init = true; diff --git a/client/app/visualizations/funnel/funnel-editor.html b/client/app/visualizations/funnel/funnel-editor.html index cb52635af2..7240fd559e 100644 --- a/client/app/visualizations/funnel/funnel-editor.html +++ b/client/app/visualizations/funnel/funnel-editor.html @@ -5,38 +5,41 @@
- +
- +
- +
- +
- +
-
+
- +
diff --git a/client/app/visualizations/funnel/index.js b/client/app/visualizations/funnel/index.js index 4b38f7cf28..7fe167cc41 100644 --- a/client/app/visualizations/funnel/index.js +++ b/client/app/visualizations/funnel/index.js @@ -1,17 +1,20 @@ -import { debounce, sortBy, isNumber, every, difference } from 'lodash'; +import { debounce, sortBy, isFinite, every, difference, merge, map } from 'lodash'; import d3 from 'd3'; import angular from 'angular'; +import { angular2react } from 'angular2react'; +import { registerVisualization } from '@/visualizations'; -import { ColorPalette, normalizeValue } from '@/visualizations/chart/plotly/utils'; +import { normalizeValue } from '@/visualizations/chart/plotly/utils'; +import ColorPalette from '@/visualizations/ColorPalette'; import editorTemplate from './funnel-editor.html'; import './funnel.less'; -function isNoneNaNNum(val) { - if (!isNumber(val) || isNaN(val)) { - return false; - } - return true; -} +const DEFAULT_OPTIONS = { + stepCol: { colName: '', displayAs: 'Steps' }, + valueCol: { colName: '', displayAs: 'Value' }, + sortKeyCol: { colName: '' }, + autoSort: true, +}; function normalizePercentage(num) { if (num < 0.01) { @@ -27,7 +30,7 @@ function Funnel(scope, element) { this.element = element; this.watches = []; const vis = d3.select(element); - const options = scope.visualization.options; + const options = scope.$ctrl.options; function drawFunnel(data) { const maxToPrevious = d3.max(data, d => d.pctPrevious); @@ -62,9 +65,7 @@ function Funnel(scope, element) { trs .append('td') .attr('class', 'col-xs-3 step') - .text(d => d.step) - .append('div') - .attr('class', 'step-name') + .attr('title', d => d.step) .text(d => d.step); // Funnel bars @@ -116,6 +117,9 @@ function Funnel(scope, element) { } function prepareData(queryData) { + if (queryData.length === 0) { + return []; + } const data = queryData.map( row => ({ step: normalizeValue(row[options.stepCol.colName]), @@ -132,7 +136,7 @@ function Funnel(scope, element) { } // Column validity - if (sortedData[0].value === 0 || !every(sortedData, d => isNoneNaNNum(d.value))) { + if (sortedData[0].value === 0 || !every(sortedData, d => isFinite(d.value))) { return; } const maxVal = d3.max(data, d => d.value); @@ -144,15 +148,12 @@ function Funnel(scope, element) { } function invalidColNames() { - const colNames = scope.queryResult.getColumnNames(); + const colNames = map(scope.$ctrl.data.columns, col => col.name); const colToCheck = [options.stepCol.colName, options.valueCol.colName]; if (!options.autoSort) { colToCheck.push(options.sortKeyCol.colName); } - if (difference(colToCheck, colNames).length > 0) { - return true; - } - return false; + return difference(colToCheck, colNames).length > 0; } function refresh() { @@ -161,79 +162,74 @@ function Funnel(scope, element) { return; } - const queryData = scope.queryResult.getData(); + const queryData = scope.$ctrl.data.rows; const data = prepareData(queryData, options); - if (data) { + if (data.length > 0) { createVisualization(data); // draw funnel } } refresh(); - this.watches.push(scope.$watch('visualization.options', refresh, true)); - this.watches.push(scope.$watch('queryResult && queryResult.getData()', refresh)); + this.watches.push(scope.$watch('$ctrl.data', refresh)); + this.watches.push(scope.$watch('$ctrl.options', refresh, true)); } Funnel.prototype.remove = function remove() { this.watches.forEach((unregister) => { unregister(); }); - angular.element(this.element).empty('.vis-container'); + angular.element(this.element).empty(); }; -function funnelRenderer() { - return { - restrict: 'E', - template: '
', - link(scope, element) { - const container = element[0].querySelector('.funnel-visualization-container'); - let funnel = new Funnel(scope, container); - - function resize() { - funnel.remove(); - funnel = new Funnel(scope, container); - } - - scope.handleResize = debounce(resize, 50); - - scope.$watch('visualization.options', (oldValue, newValue) => { - if (oldValue !== newValue) { - resize(); - } - }); - }, - }; -} +const FunnelRenderer = { + template: '
', + bindings: { + data: '<', + options: '<', + }, + controller($scope, $element) { + const container = $element[0].querySelector('.funnel-visualization-container'); + let funnel = new Funnel($scope, container); + + const update = () => { + funnel.remove(); + funnel = new Funnel($scope, container); + }; -function funnelEditor() { - return { - restrict: 'E', - template: editorTemplate, - }; -} + $scope.handleResize = debounce(update, 50); + + $scope.$watch('$ctrl.data', update); + $scope.$watch('$ctrl.options', update, true); + }, +}; + +const FunnelEditor = { + template: editorTemplate, + bindings: { + data: '<', + options: '<', + onOptionsChange: '<', + }, + controller($scope) { + $scope.$watch('$ctrl.options', (options) => { + this.onOptionsChange(options); + }, true); + }, +}; export default function init(ngModule) { - ngModule.directive('funnelRenderer', funnelRenderer); - ngModule.directive('funnelEditor', funnelEditor); - - ngModule.config((VisualizationProvider) => { - const renderTemplate = - ''; - - const editTemplate = ''; - const defaultOptions = { - stepCol: { colName: '', displayAs: 'Steps' }, - valueCol: { colName: '', displayAs: 'Value' }, - sortKeyCol: { colName: '' }, - autoSort: true, - defaultRows: 10, - }; + ngModule.component('funnelRenderer', FunnelRenderer); + ngModule.component('funnelEditor', FunnelEditor); - VisualizationProvider.registerVisualization({ + ngModule.run(($injector) => { + registerVisualization({ type: 'FUNNEL', name: 'Funnel', - renderTemplate, - editorTemplate: editTemplate, - defaultOptions, + getOptions: options => merge({}, DEFAULT_OPTIONS, options), + Renderer: angular2react('funnelRenderer', FunnelRenderer, $injector), + Editor: angular2react('funnelEditor', FunnelEditor, $injector), + + defaultRows: 10, }); }); } diff --git a/client/app/visualizations/index.js b/client/app/visualizations/index.js index 99f2e2e550..faad2d6e48 100644 --- a/client/app/visualizations/index.js +++ b/client/app/visualizations/index.js @@ -1,148 +1,94 @@ -import moment from 'moment'; -import { isArray, reduce } from 'lodash'; - -function VisualizationProvider() { - this.visualizations = {}; - // this.visualizationTypes = {}; - this.visualizationTypes = []; - const defaultConfig = { - defaultOptions: {}, - skipTypes: false, - editorTemplate: null, - }; - - this.registerVisualization = (config) => { - const visualization = Object.assign({}, defaultConfig, config); - - // TODO: this is prone to errors; better refactor. - if (this.defaultVisualization === undefined && !visualization.name.match(/Deprecated/)) { - this.defaultVisualization = visualization; - } - - this.visualizations[config.type] = visualization; - - if (!config.skipTypes) { - this.visualizationTypes.push({ name: config.name, type: config.type }); - } - }; - - this.getSwitchTemplate = (property) => { - const pattern = /(<[a-zA-Z0-9-]*?)( |>)/; - - let mergedTemplates = reduce( - this.visualizations, - (templates, visualization) => { - if (visualization[property]) { - const ngSwitch = `$1 ng-switch-when="${visualization.type}" $2`; - const template = visualization[property].replace(pattern, ngSwitch); - - return `${templates}\n${template}`; - } - - return templates; - }, - '', - ); - - mergedTemplates = `
${mergedTemplates}
`; +import { find } from 'lodash'; +import PropTypes from 'prop-types'; + +/* -------------------------------------------------------- + Types +-----------------------------------------------------------*/ + +const VisualizationOptions = PropTypes.object; // eslint-disable-line react/forbid-prop-types + +const Data = PropTypes.shape({ + columns: PropTypes.arrayOf(PropTypes.object).isRequired, + rows: PropTypes.arrayOf(PropTypes.object).isRequired, +}); + +export const VisualizationType = PropTypes.shape({ + type: PropTypes.string.isRequired, + name: PropTypes.string.isRequired, + options: VisualizationOptions.isRequired, // eslint-disable-line react/forbid-prop-types +}); + +// For each visualization's renderer +export const RendererPropTypes = { + visualizationName: PropTypes.string, + data: Data.isRequired, + options: VisualizationOptions.isRequired, + onOptionsChange: PropTypes.func, // (newOptions) => void +}; + +// For each visualization's editor +export const EditorPropTypes = { + visualizationName: PropTypes.string, + data: Data.isRequired, + options: VisualizationOptions.isRequired, + onOptionsChange: PropTypes.func.isRequired, // (newOptions) => void +}; + +/* -------------------------------------------------------- + Visualizations registry +-----------------------------------------------------------*/ + +export const registeredVisualizations = {}; + +const VisualizationConfig = PropTypes.shape({ + type: PropTypes.string.isRequired, + name: PropTypes.string.isRequired, + getOptions: PropTypes.func.isRequired, // (existingOptions: object, data: { columns[], rows[] }) => object + isDeprecated: PropTypes.bool, + Renderer: PropTypes.func.isRequired, + Editor: PropTypes.func, + + // other config options + autoHeight: PropTypes.bool, + defaultRows: PropTypes.number, + defaultColumns: PropTypes.number, + minRows: PropTypes.number, + maxRows: PropTypes.number, + minColumns: PropTypes.number, + maxColumns: PropTypes.number, +}); + +function validateVisualizationConfig(config) { + const typeSpecs = { config: VisualizationConfig }; + const values = { config }; + PropTypes.checkPropTypes(typeSpecs, values, 'prop', 'registerVisualization'); +} - return mergedTemplates; - }; +export function registerVisualization(config) { + validateVisualizationConfig(config); + config = { ...config }; // clone - this.$get = ($resource) => { - const Visualization = $resource('api/visualizations/:id', { id: '@id' }); - Visualization.visualizations = this.visualizations; - Visualization.visualizationTypes = this.visualizationTypes; - Visualization.renderVisualizationsTemplate = this.getSwitchTemplate('renderTemplate'); - Visualization.editorTemplate = this.getSwitchTemplate('editorTemplate'); - Visualization.defaultVisualization = this.defaultVisualization; + if (registeredVisualizations[config.type]) { + throw new Error(`Visualization ${config.type} already registered.`); + } - return Visualization; - }; + registeredVisualizations[config.type] = config; } -function VisualizationName(Visualization) { - return { - restrict: 'E', - scope: { - visualization: '=', - }, - template: '{{name}}', - replace: false, - link(scope) { - if (Visualization.visualizations[scope.visualization.type]) { - const defaultName = Visualization.visualizations[scope.visualization.type].name; - if (defaultName !== scope.visualization.name) { - scope.name = scope.visualization.name; - } - } - }, - }; -} +/* -------------------------------------------------------- + Helpers +-----------------------------------------------------------*/ -function VisualizationRenderer(Visualization) { - return { - restrict: 'E', - scope: { - visualization: '=', - queryResult: '=', - }, - // TODO: using switch here (and in the options editor) might introduce errors and bad - // performance wise. It's better to eventually show the correct template based on the - // visualization type and not make the browser render all of them. - template: `\n${Visualization.renderVisualizationsTemplate}`, - replace: false, - link(scope) { - scope.$watch('queryResult && queryResult.getFilters()', (filters) => { - if (filters) { - scope.filters = filters; - } - }); - }, - }; +export function getDefaultVisualization() { + return find(registeredVisualizations, visualization => !visualization.isDeprecated); } -function VisualizationOptionsEditor(Visualization) { +export function newVisualization(type = null, options = {}) { + const visualization = type ? registeredVisualizations[type] : getDefaultVisualization(); return { - restrict: 'E', - template: Visualization.editorTemplate, - replace: false, - scope: { - visualization: '=', - query: '=', - queryResult: '=', - }, + type: visualization.type, + name: visualization.name, + description: '', + options, }; } - -function FilterValueFilter(clientConfig) { - return (value, filter) => { - let firstValue = value; - if (isArray(value)) { - firstValue = value[0]; - } - - // TODO: deduplicate code with table.js: - if (filter.column.type === 'date') { - if (firstValue && moment.isMoment(firstValue)) { - return firstValue.format(clientConfig.dateFormat); - } - } else if (filter.column.type === 'datetime') { - if (firstValue && moment.isMoment(firstValue)) { - return firstValue.format(clientConfig.dateTimeFormat); - } - } - - return firstValue; - }; -} - -export default function init(ngModule) { - ngModule.provider('Visualization', VisualizationProvider); - ngModule.directive('visualizationRenderer', VisualizationRenderer); - ngModule.directive('visualizationOptionsEditor', VisualizationOptionsEditor); - ngModule.directive('visualizationName', VisualizationName); - ngModule.filter('filterValue', FilterValueFilter); -} - -init.init = true; diff --git a/client/app/visualizations/map/index.js b/client/app/visualizations/map/index.js index e7b799bfe8..96268107d8 100644 --- a/client/app/visualizations/map/index.js +++ b/client/app/visualizations/map/index.js @@ -10,13 +10,13 @@ import markerIconRetina from 'leaflet/dist/images/marker-icon-2x.png'; import markerShadow from 'leaflet/dist/images/marker-shadow.png'; import 'leaflet-fullscreen'; import 'leaflet-fullscreen/dist/leaflet.fullscreen.css'; +import { angular2react } from 'angular2react'; +import { registerVisualization } from '@/visualizations'; import template from './map.html'; import editorTemplate from './map-editor.html'; -/* -This is a workaround for an issue with giving Leaflet load the icon on its own. -*/ +// This is a workaround for an issue with giving Leaflet load the icon on its own. L.Icon.Default.mergeOptions({ iconUrl: markerIcon, iconRetinaUrl: markerIconRetina, @@ -25,285 +25,316 @@ L.Icon.Default.mergeOptions({ delete L.Icon.Default.prototype._getIconUrl; -function mapRenderer() { - return { - restrict: 'E', - template, - link($scope, elm) { - const colorScale = d3.scale.category10(); - const map = L.map(elm[0].children[0].children[0], { - scrollWheelZoom: false, - fullscreenControl: true, - }); - const mapControls = L.control.layers().addTo(map); - const layers = {}; - const tileLayer = L.tileLayer('//{s}.tile.openstreetmap.org/{z}/{x}/{y}.png', { - attribution: '© OpenStreetMap contributors', - }).addTo(map); - - function getBounds() { - $scope.visualization.options.bounds = map.getBounds(); - } - - function setBounds() { - const b = $scope.visualization.options.bounds; +const MAP_TILES = [ + { + name: 'OpenStreetMap', + url: '//{s}.tile.openstreetmap.org/{z}/{x}/{y}.png', + }, + { + name: 'OpenStreetMap BW', + url: '//{s}.tiles.wmflabs.org/bw-mapnik/{z}/{x}/{y}.png', + }, + { + name: 'OpenStreetMap DE', + url: '//{s}.tile.openstreetmap.de/tiles/osmde/{z}/{x}/{y}.png', + }, + { + name: 'OpenStreetMap FR', + url: '//{s}.tile.openstreetmap.fr/osmfr/{z}/{x}/{y}.png', + }, + { + name: 'OpenStreetMap Hot', + url: '//{s}.tile.openstreetmap.fr/hot/{z}/{x}/{y}.png', + }, + { + name: 'Thunderforest', + url: '//{s}.tile.thunderforest.com/cycle/{z}/{x}/{y}.png', + }, + { + name: 'Thunderforest Spinal', + url: '//{s}.tile.thunderforest.com/spinal-map/{z}/{x}/{y}.png', + }, + { + name: 'OpenMapSurfer', + url: '//korona.geog.uni-heidelberg.de/tiles/roads/x={x}&y={y}&z={z}', + }, + { + name: 'Stamen Toner', + url: '//stamen-tiles-{s}.a.ssl.fastly.net/toner/{z}/{x}/{y}.png', + }, + { + name: 'Stamen Toner Background', + url: '//stamen-tiles-{s}.a.ssl.fastly.net/toner-background/{z}/{x}/{y}.png', + }, + { + name: 'Stamen Toner Lite', + url: '//stamen-tiles-{s}.a.ssl.fastly.net/toner-lite/{z}/{x}/{y}.png', + }, + { + name: 'OpenTopoMap', + url: '//{s}.tile.opentopomap.org/{z}/{x}/{y}.png', + }, +]; + +const DEFAULT_OPTIONS = { + classify: 'none', + clusterMarkers: true, +}; + +function heatpoint(lat, lon, color) { + const style = { + fillColor: color, + fillOpacity: 0.9, + stroke: false, + }; - if (b) { - map.fitBounds([[b._southWest.lat, b._southWest.lng], - [b._northEast.lat, b._northEast.lng]]); - } else if (layers) { - const allMarkers = _.flatten(_.map(_.values(layers), l => l.getLayers())); - // eslint-disable-next-line new-cap - const group = new L.featureGroup(allMarkers); - map.fitBounds(group.getBounds()); - } - } + return L.circleMarker([lat, lon], style); +} +const createMarker = (lat, lon) => L.marker([lat, lon]); - map.on('focus', () => { map.on('moveend', getBounds); }); - map.on('blur', () => { map.off('moveend', getBounds); }); +function createDescription(latCol, lonCol, row) { + const lat = row[latCol]; + const lon = row[lonCol]; - function resize() { - if (!map) return; - map.invalidateSize(false); - setBounds(); - } + let description = '