diff --git a/.gitignore b/.gitignore index 7eb7b6d..2d3e86d 100644 --- a/.gitignore +++ b/.gitignore @@ -33,3 +33,6 @@ nubis/terraform/.terraform nubis/terraform/terraform.tfvars nubis/builder/artifacts/*-dev/ nubis/builder/artifacts/AMIs.json + +# Database dump for local development +wiki.sql* \ No newline at end of file diff --git a/gcp/Dockerfile b/gcp/Dockerfile new file mode 100644 index 0000000..5b99337 --- /dev/null +++ b/gcp/Dockerfile @@ -0,0 +1,37 @@ +# Ensure both post 1. numbers match here. +FROM docker.io/mediawiki:1.39 +ENV MWIKI_VER=39 +WORKDIR /var/www/html/ + +ARG UID=10001 +ARG GID=10001 + +RUN apt-get update && apt-get install -y --no-install-recommends ffmpeg unzip + +# Prepare for nonroot user +RUN groupadd -g $GID app; \ + useradd -g $GID -u $UID -m -s /usr/sbin/nologin app; \ + chown -R app:app /var/www/html/ + +USER app + +RUN git clone --depth 1 --single-branch --branch REL1_${MWIKI_VER} https://gerrit.wikimedia.org/r/mediawiki/extensions/ConfirmAccount /var/www/html/extensions/ConfirmAccount +RUN git clone --depth 1 --single-branch --branch REL1_${MWIKI_VER} https://gerrit.wikimedia.org/r/mediawiki/extensions/LabeledSectionTransclusion /var/www/html/extensions/LabeledSectionTransclusion +RUN git clone --depth 1 --single-branch --branch REL1_${MWIKI_VER} https://gerrit.wikimedia.org/r/mediawiki/extensions/TimedMediaHandler /var/www/html/extensions/TimedMediaHandler +RUN git clone --depth 1 --single-branch --branch REL1_${MWIKI_VER} https://gerrit.wikimedia.org/r/mediawiki/extensions/RSS /var/www/html/extensions/RSS +RUN git clone --depth 1 --single-branch --branch REL1_${MWIKI_VER} https://gerrit.wikimedia.org/r/mediawiki/extensions/PageForms /var/www/html/extensions/PageForms +RUN git clone --depth 1 --single-branch --branch REL1_${MWIKI_VER} https://gerrit.wikimedia.org/r/mediawiki/extensions/UrlGetParameters /var/www/html/extensions/UrlGetParameters +RUN git clone --depth 1 --single-branch --branch REL1_${MWIKI_VER} https://gerrit.wikimedia.org/r/mediawiki/extensions/NoTitle /var/www/html/extensions/NoTitle +RUN git clone --depth 1 --single-branch --branch REL1_${MWIKI_VER} https://gerrit.wikimedia.org/r/mediawiki/extensions/Widgets /var/www/html/extensions/Widgets +RUN git clone --depth 1 --single-branch --branch REL1_${MWIKI_VER} https://gerrit.wikimedia.org/r/mediawiki/extensions/MobileFrontend /var/www/html/extensions/MobileFrontend +RUN git clone --depth 1 --single-branch --branch main https://github.com/mozilla/mediawiki-bugzilla.git /var/www/html/extensions/Bugzilla + +RUN php -r "copy('https://getcomposer.org/installer', 'composer-setup.php');" && \ + php -r "if (hash_file('sha384', 'composer-setup.php') === 'dac665fdc30fdd8ec78b38b9800061b4150413ff2e3b6f88543c636f7cd84f6db9189d43a81e5503cda447da73c7e5b6') { echo 'Installer verified'; } else { echo 'Installer corrupt'; unlink('composer-setup.php'); } echo PHP_EOL;" && \ + php composer-setup.php && \ + php -r "unlink('composer-setup.php');" + +COPY composer.local.json /var/www/html/composer.local.json +COPY ports.conf /etc/apache2/ports.conf +COPY LocalSettings.php /var/www/html/LocalSettings.php +RUN php composer.phar update --no-dev diff --git a/gcp/LocalSettings.php b/gcp/LocalSettings.php new file mode 100644 index 0000000..e045e15 --- /dev/null +++ b/gcp/LocalSettings.php @@ -0,0 +1,551 @@ + "Mozilla2", + 101 => "Mozilla2_Talk", + 102 => "Calendar", + 103 => "Calendar_Talk", + 106 => "Gecko", + 107 => "Gecko_Talk", + 108 => "PluginFutures", + 109 => "PluginFutures_Talk", + 110 => "SVG", + 111 => "SVG_Talk", + 112 => "XUL", + 113 => "XUL_Talk", + 114 => "L10n", + 115 => "L10n_Talk", + 116 => "Update", + 117 => "Update_Talk", + 118 => "SVGDev", + 119 => "SVGDev_Talk", + 122 => "Bugzilla", + 123 => "Bugzilla_Talk", + 128 => "MailNews", + 129 => "MailNews_Talk", + # 132 - 141 - Semantic MediaWiki extension + ); + +$wgNamespacesToBeSearchedDefault = array( + -1 => 0, 1 => 0, 2 => 0, 3 => 0, 4 => 0, 5 => 0, 6 => 0, 7 => 0, 8 => 0, 10 => 0, + 0 => 1, 9 => 1, 11 => 1, + 100 => 1, 101 => 1, + 102 => 1, 103 => 1, + 104 => 1, 105 => 1, + 106 => 1, 107 => 1, + 108 => 1, 109 => 1, + 110 => 1, 111 => 1, + 112 => 1, 113 => 1, + 114 => 1, 115 => 1, + 116 => 1, 117 => 1, + 118 => 1, 119 => 1, + 120 => 1, 121 => 1, + 122 => 1, 123 => 1, + 124 => 1, 125 => 1, + 126 => 1, 127 => 1, + 128 => 1, 129 => 1, + 130 => 1, 131 => 1, +); + +# Enable subpages in all namespaces +$wgNamespacesWithSubpages = array_fill(0, 200, true); + +## Default skin: you can change the default skin. Use the internal symbolic +## names, ie 'vector', 'monobook': +$wgDefaultSkin = "vector"; + +# Enabled skins. +# The following skins were automatically enabled: +#wfLoadSkin( 'CologneBlue' ); +#wfLoadSkin( 'Modern' ); +wfLoadSkin( 'MonoBook' ); +wfLoadSkin( 'Vector' ); + +# End of automatically generated settings. +# Add more configuration options below. + +# May need to disable some/all of these as they are only included with default installs somewhere in 1.3x. + +# Bug 721366 and 731672 +require_once("$IP/extensions/Bugzilla/Bugzilla.php"); +//wfLoadExtension( 'Bugzilla' ); +wfLoadExtension( 'ConfirmAccount' ); +$wgConfirmAccountRequestFormItems['Biography'] = array( 'enabled' => true, 'minWords' => 15 ); +$wgUseRealNamesOnly = false; +$wgAccountRequestToS = true; +$wgAllowAccountRequestFiles = false; +$wgAccountRequestThrottle = 50; +$wgConfirmAccountSaveInfo = false; +$wgConfirmAccountCaptchas = true; + +wfLoadExtensions( [ 'ConfirmEdit', 'ConfirmEdit/QuestyCaptcha' ] ); +$wgCaptchaTriggers['edit'] = false; +$wgCaptchaTriggers['create'] = false; +$wgCaptchaTriggers['addurl'] = false; +$wgCaptchaTriggers['createaccount'] = true; +$wgCaptchaTriggers['badlogin'] = true; +$wgCaptchaClass = 'QuestyCaptcha'; + +$question1 = getenv("question1") ? getenv("question1") : random_bytes(30); +$answer1 = getenv("answer1") ? getenv("answer1") : random_bytes(30); +$question2 = getenv("question2") ? getenv("question2") : random_bytes(30); +$answer2 = getenv("answer2") ? getenv("answer2") : random_bytes(30); +$question3 = getenv("question3") ? getenv("question3") : random_bytes(30); +$answer3 = getenv("answer3") ? getenv("answer3") : random_bytes(30); +$wgCaptchaQuestions = [ + $question1 => $answer1, + $question2 => $answer2, + $question3 => $answer3, +]; + +wfLoadExtension( 'Gadgets' ); +wfLoadExtension( 'ImageMap' ); +wfLoadExtension( 'Interwiki' ); +wfLoadExtension( 'LabeledSectionTransclusion' ); +wfLoadExtension( 'Nuke' ); +wfLoadExtension( 'TimedMediaHandler'); +wfLoadExtension( 'ParserFunctions' ); +wfLoadExtension( 'Renameuser' ); + +wfLoadExtension( 'RSS' ); +$wgRSSUrlWhitelist = array( 'http://benjamin.smedbergs.us/weekly-updates.fcgi/project/firefox/feed', + 'http://blog.wikimedia.org/feed/', + 'https://blog.mozilla.org/feed/', + 'https://hacks.mozilla.org/feed/', + 'https://quality.mozilla.org/feed/', + 'https://blog.lizardwrangler.com/feed/', + 'https://brendaneich.com/feed/', + ); + +wfLoadExtension( 'PageForms' ); + +wfLoadExtension( 'SpamBlacklist' ); +$wgSpamBlacklistFiles = array( + // database title + "DB: $wgDBname Spam_blacklist", + ); + +wfLoadExtension( 'SyntaxHighlight_GeSHi' ); + +wfLoadExtension( 'WikiEditor' ); +# Enables use of WikiEditor by default but still allow users to disable it in preferences +$wgDefaultUserOptions['usebetatoolbar'] = 1; +$wgDefaultUserOptions['usebetatoolbar-cgd'] = 1; +# Displays the Preview and Changes tabs +$wgDefaultUserOptions['wikieditor-preview'] = 1; +// Make default the user option to prompt for an edit summary if none is provided +// does not affect users who have already set this option +// bug 1080898 +$wgDefaultUserOptions['forceeditsummary'] = true; + +wfLoadExtension( 'NoTitle' ); +wfLoadExtension( 'InputBox' ); +wfLoadExtension( 'Widgets' ); +wfLoadExtension( 'MobileFrontend' ); + +// Disable for 1.35 upgrade +if (getenv("MWIKI_VER") != "35") { + wfLoadExtension('SubPageList'); + wfLoadExtension('UrlGetParameters'); + + wfLoadSkin( 'MinervaNeue' ); + $wgDefaultMobileSkin = 'minerva'; +} + +$wgLogos = [ + 'icon' => "$wgUploadPath/mozilla-wiki-logo-alt-135px.png", + '1x' => "$wgUploadPath/mozilla-wiki-logo-alt-135px.png" +]; + +######### Bug 397718 ############ +$wgMimeDetectorCommand= "file -bi"; #use external mime detector (Linux) +################################# + +$wgAllowExternalImages = true; + +$wgSitename = "MozillaWiki"; + +# The relative URL path to the favicon +$wgFavicon = "$wgUploadPath/favicon.ico"; + +$wgShowExceptionDetails = true; + +$wgMemoryLimit = "256M"; + +$wgShowIPinHeader = false; +$wgFileExtensions = array( 'gz', 'tar', 'png', 'gif', 'jpg', 'jpeg', 'ppt', 'pdf', 'doc', 'xls', 'zip', 'ics', 'mp3', 'ogg', 'odt', 'odp', 'svg', 'odt', 'ods', 'odg', 'webm' ); + +$wgAllowTitlesInSVG = true; + +// Don't convert, just serve and let the browser render/save/whatever +$wgSVGConverter = 'rsvg'; + +$wgWhitelistRead = array( 'Main Page', 'Special:Userlogin', 'Special:Userlogout', '-', 'MediaWiki:Monobook.css', 'MediaWiki:Monobook.js' ); + +$wgAutoConfirmAge = 5 * 3600 * 24; // 5 days to pass isNewbie() +$wgAutoConfirmCount = 10; // and have ten edits + +$wgRCMaxAge = 31536000; // one year + +# Controls the title displayed by subpages +$wgRestrictDisplayTitle = false; + +// Implicitly add all users to 'inactive' group whose accounts: +// * are older than 6 months, and +// * have less than 1 edit. +// don't wipe out existing autopromote autoconfirm +$wgAutopromote['inactive'] = array( '&', + array( APCOND_AGE, 60 * 60 * 24 * 30 * 6 ), + array( '!', array( APCOND_EDITCOUNT, 1 ) ), +); + +// TRUE in this case revokes the permission. +$wgRevokePermissions['inactive']['createpage'] = true; +$wgRevokePermissions['inactive']['createtalk'] = true; +$wgRevokePermissions['inactive']['move'] = true; +$wgRevokePermissions['inactive']['movefile'] = true; +$wgRevokePermissions['inactive']['move-subpages'] = true; +$wgRevokePermissions['inactive']['upload'] = true; +$wgRevokePermissions['inactive']['reupload'] = true; +$wgRevokePermissions['inactive']['reupload-own'] = true; + +// Bug 1082298 +$wgPFEnableStringFunctions = true; + +// must disable jquery table on legacy mediawiki-bugzilla extension for mobile editing to work +$wgBugzillaJqueryTable = false; + +// +if (getenv("UPGRADE_MODE") == "enabled") { + $wgReadOnly = ( PHP_SAPI === 'cli' ) ? false : 'This wiki is currently being upgraded to a newer software version. Please check back soon.'; +} diff --git a/gcp/README.md b/gcp/README.md new file mode 100644 index 0000000..05a0592 --- /dev/null +++ b/gcp/README.md @@ -0,0 +1,38 @@ +# wikimo test + +## "Runbook" + +Use `docker compose up --build mediawiki` after changing the version in `Dockerfile`. + +To migrate the db dump to the latest version use the following path: + +- switch the version in `Dockerfile` to `mediawiki:1.35` and update MWIKI_VER to match the point release number. +- Set "wikimedia/at-ease": "v2.1.0" to "wikimedia/at-ease": "v2.0.0" +- Run `docker compose exec mediawiki php maintenance/update.php` + +- switch the version in `Dockerfile` to `mediawiki:1.39` and update MWIKI_VER to match the point release number. +- Set "wikimedia/at-ease": "v2.0.0" to "wikimedia/at-ease": "v2.1.0" +- Run `docker compose exec mediawiki php maintenance/update.php` + +### Notes +- switch the version in `Dockerfile` to `mediawiki:1.xx` and update MWIKI_VER to match the point release number. +- Run `docker-compose exec mediawiki php maintenance/run.php update.php` + +## Cleanup Scripts +Scripts that could be run on >=1.27 +```shell +compose exec mediawiki php maintenance/deleteArchivedRevisions.php --delete +compose exec mediawiki php maintenance/removeUnusedAccounts.php --delete +``` + +## Migration Plan +1. Put the AWS Wiki into maintenance. +2. Dump the DB. +3. Import DB and images into GCP environment. +4. Deploy a 1.35 build to GCP. +5. Run `php maintenance/update.php` with 1.35. +6. Deploy a 1.39 build to GCP +7. Check GCP version of wiki. +8. Update DNS to point to GCP Wiki. +9. Remove maintenance mode on GCP Wiki. +10. Clean up AWS Wiki (and optionally Nubis). \ No newline at end of file diff --git a/gcp/composer.local.json b/gcp/composer.local.json new file mode 100644 index 0000000..37cf802 --- /dev/null +++ b/gcp/composer.local.json @@ -0,0 +1,18 @@ +{ + "config": { + "allow-plugins": true + }, + "require": { + "mediawiki/sub-page-list": "~3.0", + "wikimedia/normalized-exception": "v1.0.1", + "wikimedia/at-ease": "v2.1.0" + }, + "extra": { + "merge-plugin": { + "include": [ + "extensions/TimedMediaHandler/composer.json", + "extensions/Widgets/composer.json" + ] + } + } +} \ No newline at end of file diff --git a/gcp/docker-compose.yaml b/gcp/docker-compose.yaml new file mode 100644 index 0000000..04789f8 --- /dev/null +++ b/gcp/docker-compose.yaml @@ -0,0 +1,24 @@ +services: + mediawiki: + build: . + ports: + - 8080:80 + volumes: + - ./images:/var/www/html/images + - ./LocalSettings.php:/var/www/html/LocalSettings.php + - ../mediawiki-bugzilla/:/var/www/html/extensions/Bugzilla # can be removed once our changes are upstream + db: + image: docker.io/mariadb:11 + restart: always + environment: + MYSQL_DATABASE: 'db' + MYSQL_USER: 'user' + MYSQL_PASSWORD: 'password' + MYSQL_ROOT_PASSWORD: 'password' + ports: + - '127.0.0.1:3306:3306' + volumes: + - data:/var/lib/mysql + - ./wiki.sql:/docker-entrypoint-initdb.d/wiki.sql +volumes: + data: diff --git a/gcp/podman-compose.yaml b/gcp/podman-compose.yaml new file mode 100644 index 0000000..77ab6ca --- /dev/null +++ b/gcp/podman-compose.yaml @@ -0,0 +1,23 @@ +services: + mediawiki: + image: docker.io/mediawiki:1.39 + ports: + - 8080:80 + volumes: + - /var/www/html/images + - ./LocalSettings.php:/var/www/html/LocalSettings.php + db: + image: docker.io/mariadb:11 + restart: always + environment: + MYSQL_DATABASE: 'db' + MYSQL_USER: 'user' + MYSQL_PASSWORD: 'password' + MYSQL_ROOT_PASSWORD: 'password' + ports: + - '127.0.0.1:3306:3306' + volumes: + - data:/var/lib/mysql + - ./wiki.sql:/docker-entrypoint-initdb.d/wiki.sql +volumes: + data: diff --git a/gcp/ports.conf b/gcp/ports.conf new file mode 100644 index 0000000..afbac3b --- /dev/null +++ b/gcp/ports.conf @@ -0,0 +1,13 @@ +# If you just change the port or add more ports here, you will likely also +# have to change the VirtualHost statement in +# /etc/apache2/sites-enabled/000-default.conf + +Listen 8000 + + + Listen 443 + + + + Listen 443 + \ No newline at end of file diff --git a/gcp/test/.ruby-version b/gcp/test/.ruby-version new file mode 100644 index 0000000..bea438e --- /dev/null +++ b/gcp/test/.ruby-version @@ -0,0 +1 @@ +3.3.1 diff --git a/gcp/test/Gemfile b/gcp/test/Gemfile new file mode 100644 index 0000000..f339105 --- /dev/null +++ b/gcp/test/Gemfile @@ -0,0 +1,7 @@ +# frozen_string_literal: true + +source 'https://rubygems.org' + +gem 'nokogiri', '~> 1.16' + +gem 'typhoeus', '~> 1.4' diff --git a/gcp/test/Gemfile.lock b/gcp/test/Gemfile.lock new file mode 100644 index 0000000..4345c15 --- /dev/null +++ b/gcp/test/Gemfile.lock @@ -0,0 +1,41 @@ +GEM + remote: https://rubygems.org/ + specs: + ethon (0.16.0) + ffi (>= 1.15.0) + ffi (1.17.0-aarch64-linux-gnu) + ffi (1.17.0-arm-linux-gnu) + ffi (1.17.0-arm64-darwin) + ffi (1.17.0-x86-linux-gnu) + ffi (1.17.0-x86_64-darwin) + ffi (1.17.0-x86_64-linux-gnu) + nokogiri (1.16.7-aarch64-linux) + racc (~> 1.4) + nokogiri (1.16.7-arm-linux) + racc (~> 1.4) + nokogiri (1.16.7-arm64-darwin) + racc (~> 1.4) + nokogiri (1.16.7-x86-linux) + racc (~> 1.4) + nokogiri (1.16.7-x86_64-darwin) + racc (~> 1.4) + nokogiri (1.16.7-x86_64-linux) + racc (~> 1.4) + racc (1.8.1) + typhoeus (1.4.1) + ethon (>= 0.9.0) + +PLATFORMS + aarch64-linux + arm-linux + arm64-darwin + x86-linux + x86_64-darwin + x86_64-linux + +DEPENDENCIES + nokogiri (~> 1.16) + typhoeus (~> 1.4) + +BUNDLED WITH + 2.5.9 diff --git a/gcp/test/test.rb b/gcp/test/test.rb new file mode 100644 index 0000000..af0dfff --- /dev/null +++ b/gcp/test/test.rb @@ -0,0 +1,50 @@ +# frozen_string_literal: true + +require 'typhoeus' +require 'nokogiri' + +BASE_URL_PROD = 'https://wiki.mozilla.org' +BASE_URL_TEST = 'http://localhost:8080/index.php' + +INSPECT_ELEMENT = '#content' + +TEST_PAGES_COUNT = 1000 +TEST_KEYWORDS = ['Warning: ', 'Error: '].freeze + +def fetch_page(url) + response = Typhoeus.get(url, followlocation: true) + url = response.effective_url + html = response.response_body + code = response.response_code + + puts "WARN: #{url} Status code is #{code}" unless code == 200 + + [html, url] +end + +def parse_html(html) + parsed_data = Nokogiri::HTML.parse(html) + + parsed_data.css(INSPECT_ELEMENT).inner_text +end + +def pages_equal?(prod_page, test_page, keyword) + prod_has_keyword = prod_page.include?(keyword) + test_has_keyword = test_page.include?(keyword) + + puts "NOK: #{url} (#{prod_has_keyword})" if prod_has_keyword != test_has_keyword +end + +TEST_PAGES_COUNT.times do |i| + html, url = fetch_page("#{BASE_URL_PROD}/Special:Random") + prod_page = parse_html(html) + + html, = fetch_page(url.gsub(BASE_URL_PROD, BASE_URL_TEST)) + test_page = parse_html(html) + + print "\r#{i}" + + TEST_KEYWORDS.each do |keyword| + pages_equal?(prod_page, test_page, keyword) + end +end