diff --git a/.env b/.env index 220646b6..18ab1c39 100644 --- a/.env +++ b/.env @@ -19,3 +19,7 @@ NEXT_PUBLIC_API_KEY_GRAPHHOPPER=f189b841-6529-46c6-8a91-51f17477dcda # Umami anylytics is used to track server load only. We don't want to track clicks in browser. # optional, fill blank to disable UMAMI_WEBSITE_ID=5e4d4917-9031-42f1-a26a-e71d7ab8e3fe + +# Wikimedia Commons upload bot, experimental. Please leave blank. +NEXT_PUBLIC_ENABLE_UPLOAD= +OSMAPPBOT_PASSWORD= diff --git a/package.json b/package.json index 6968d43e..e1fe3a25 100644 --- a/package.json +++ b/package.json @@ -42,6 +42,7 @@ "dice-coefficient": "^2.1.1", "image-size": "^1.1.1", "isomorphic-unfetch": "^4.0.2", + "exifr": "^7.1.3", "isomorphic-xml2js": "^0.1.3", "js-cookie": "^3.0.5", "js-md5": "^0.8.3", diff --git a/pages/api/_fixture.ts b/pages/api/_fixture.ts new file mode 100644 index 00000000..25e458f0 --- /dev/null +++ b/pages/api/_fixture.ts @@ -0,0 +1,45 @@ +export const exampleUploadResponse = { + upload: { + filename: 'File_1.jpg', + result: 'Success', + imageinfo: { + url: 'https://upload.wikimedia.org/wikipedia/test/3/39/File_1.jpg', + html: '

A file with this name exists already, please check File:File 1.jpg if you are not sure if you want to change it.\n

\n
File:File 1.jpg
\n', + width: 474, + size: 26703, + bitdepth: 8, + mime: 'image/jpeg', + userid: 42588, + mediatype: 'BITMAP', + descriptionurl: 'https://test.wikipedia.org/wiki/File:File_1.jpg', + extmetadata: { + ObjectName: { + value: 'File 1', + hidden: '', + source: 'mediawiki-metadata', + }, + DateTime: { + value: '2019-03-06 08:43:37', + hidden: '', + source: 'mediawiki-metadata', + }, + // ... + }, + comment: '', + commonmetadata: [], + descriptionshorturl: 'https://test.wikipedia.org/w/index.php?curid=0', + sha1: '2ffadd0da73fab31a50407671622fd6e5282d0cf', + parsedcomment: '', + metadata: [ + { + name: 'MEDIAWIKI_EXIF_VERSION', + value: 2, + }, + ], + canonicaltitle: 'File:File 1.jpg', + user: 'Mansi29ag', + timestamp: '2019-03-06T08:43:37Z', + height: 296, + }, + }, +}; diff --git a/pages/api/upload.ts b/pages/api/upload.ts new file mode 100644 index 00000000..cf88bf95 --- /dev/null +++ b/pages/api/upload.ts @@ -0,0 +1,52 @@ +import type { NextApiRequest, NextApiResponse } from 'next'; +import { serverFetchOsmUser } from '../../src/server/osmApiAuthServer'; +import { + fetchFeatureWithCenter, + fetchParentFeatures, +} from '../../src/services/osmApi'; +import { intl, setIntl } from '../../src/services/intl'; +import { getExifData } from '../../src/server/upload/getExifData'; +import { uploadToWikimediaCommons } from '../../src/server/upload/uploadToWikimediaCommons'; +import { getApiId } from '../../src/services/helpers'; +import { File } from '../../src/server/upload/types'; +import { setProjectForSSR } from '../../src/services/project'; +import { fetchToFile } from '../../src/server/upload/fetchToFile'; +import { OsmId } from '../../src/services/types'; +import { isClimbingRoute } from '../../src/utils'; +import { Feature } from '../../src/services/types'; + +// inspiration: https://commons.wikimedia.org/wiki/File:Drive_approaching_the_Grecian_Lodges_-_geograph.org.uk_-_5765640.jpg +// https://github.com/multichill/toollabs/blob/master/bot/commons/geograph_uploader.py +// TODO https://commons.wikimedia.org/wiki/Template:Geograph_from_structured_data + +const getFeature = async (apiId: OsmId): Promise => { + const feature = await fetchFeatureWithCenter(apiId); + if (isClimbingRoute(feature)) { + const parentFeatures = await fetchParentFeatures(feature.osmMeta); + return { ...feature, parentFeatures }; + } + return feature; +}; + +export default async (req: NextApiRequest, res: NextApiResponse) => { + try { + const { shortId, lang, url, filename } = JSON.parse(req.body); + setIntl({ lang, messages: [] }); + setProjectForSSR(req); + + const apiId = getApiId(shortId); + const feature = await getFeature(apiId); + console.log('intl', intl); // eslint-disable-line no-console + const user = await serverFetchOsmUser(req.cookies.osmAccessToken); + + const filepath = await fetchToFile(url); + const { location, date } = await getExifData(filepath); + const file: File = { filepath, filename, location, date }; + const out = await uploadToWikimediaCommons(user, feature, file, lang); + + res.status(200).json(out); + } catch (err) { + console.error(err); // eslint-disable-line no-console + res.status(err.code ?? 400).send(String(err)); + } +}; diff --git a/src/components/FeaturePanel/FeaturePanel.tsx b/src/components/FeaturePanel/FeaturePanel.tsx index bb5b726b..98039d02 100644 --- a/src/components/FeaturePanel/FeaturePanel.tsx +++ b/src/components/FeaturePanel/FeaturePanel.tsx @@ -27,6 +27,7 @@ import { ClimbingStructuredData } from './Climbing/ClimbingStructuredData'; import { isPublictransportRoute } from '../../utils'; import { Sockets } from './Sockets/Sockets'; import { ClimbingTypeBadge } from './Climbing/ClimbingTypeBadge'; +import { UploadDialog } from './UploadDialog/UploadDialog'; const Flex = styled.div` flex: 1; @@ -80,6 +81,12 @@ export const FeaturePanel = ({ headingRef }: FeaturePanelProps) => { + {process.env.NEXT_PUBLIC_ENABLE_UPLOAD && ( + + + + )} + diff --git a/src/components/FeaturePanel/UploadDialog/UploadDialog.tsx b/src/components/FeaturePanel/UploadDialog/UploadDialog.tsx new file mode 100644 index 00000000..89a8b5f3 --- /dev/null +++ b/src/components/FeaturePanel/UploadDialog/UploadDialog.tsx @@ -0,0 +1,168 @@ +import React, { ChangeEvent, useState } from 'react'; +import { Button, CircularProgress } from '@mui/material'; +import { fetchJson, fetchText } from '../../../services/fetch'; +import { useFeatureContext } from '../../utils/FeatureContext'; +import { getShortId } from '../../../services/helpers'; +import { + editOsmFeature, + loginAndfetchOsmUser, +} from '../../../services/osmApiAuth'; +import { intl } from '../../../services/intl'; +import { Feature } from '../../../services/types'; +import { clearFeatureCache, quickFetchFeature } from '../../../services/osmApi'; +import { + getNextWikimediaCommonsIndex, + getWikimediaCommonsKey, +} from '../Climbing/utils/photo'; +import { useSnackbar } from '../../utils/SnackbarContext'; +import { useRouter } from 'next/navigation'; + +const WIKIPEDIA_LIMIT = 100 * 1024 * 1024; +const BUCKET_URL = 'https://osmapp-upload-tmp.s3.amazonaws.com/'; + +// Vercel has limit 4.5 MB on payload size, so we have to upload to S3 first +const uploadToS3 = async (file: File) => { + const key = `${Math.random()}/${file.name}`; + + const body = new FormData(); + body.append('key', key); + body.append('file', file); + + await fetchText(BUCKET_URL, { method: 'POST', body }); + + return `${BUCKET_URL}${key}`; +}; + +const submitToWikimediaCommons = async ( + url: string, + filename: string, + feature: Feature, +) => { + const shortId = getShortId(feature.osmMeta); + + return await fetchJson('/api/upload', { + method: 'POST', + body: JSON.stringify({ url, filename, shortId, lang: intl.lang }), + }); +}; + +const performUploadWithLogin = async ( + url: string, + filename: string, + feature: Feature, +) => { + try { + return await submitToWikimediaCommons(url, filename, feature); + } catch (e) { + if (e.code === '401') { + await loginAndfetchOsmUser(); + return await submitToWikimediaCommons(url, filename, feature); + } + throw e; + } +}; + +const submitToOsm = async (feature: Feature, fileTitle: string) => { + clearFeatureCache(feature.osmMeta); + const freshFeature = await quickFetchFeature(feature.osmMeta); + const newPhotoIndex = getNextWikimediaCommonsIndex(freshFeature.tags); + await editOsmFeature( + freshFeature, + `Upload image ${fileTitle}`, + { + ...freshFeature.tags, + [getWikimediaCommonsKey(newPhotoIndex)]: fileTitle, + }, + false, + ); + + clearFeatureCache(feature.osmMeta); +}; + +const useGetHandleFileUpload = ( + feature: Feature, + setUploading: React.Dispatch>, + setResetKey: React.Dispatch>, +) => { + const { showToast } = useSnackbar(); + const router = useRouter(); + + return async (e: ChangeEvent) => { + const file = e.target.files?.[0]; + const filename = file.name; + + if (!file) { + return; + } + + if (file.size > WIKIPEDIA_LIMIT) { + alert('Maximum file size for Wikipedia is 100 MB.'); // eslint-disable-line no-alert + return; + } + + try { + setUploading(true); + const url = await uploadToS3(file); + const wikiResponse = await performUploadWithLogin(url, filename, feature); + const osmResponse = await submitToOsm(feature, wikiResponse.title); + + showToast('Image uploaded successfully', 'success'); + router.refresh(); + } finally { + setUploading(false); + setResetKey((key) => key + 1); + } + }; +}; + +const UploadButton = () => { + const { feature } = useFeatureContext(); + const [uploading, setUploading] = useState(false); + const [resetKey, setResetKey] = useState(0); + + const handleFileUpload = useGetHandleFileUpload( + feature, + setUploading, + setResetKey, + ); + + return ( + <> + + + ); +}; + +export const UploadDialog = () => { + const { feature } = useFeatureContext(); + const { osmMeta, skeleton } = feature; + const editEnabled = !skeleton; + + return ( + <> + {editEnabled && ( + <> + +
+ + )} + + ); +}; diff --git a/src/helpers/featureLabel.ts b/src/helpers/featureLabel.ts index 7d433cc5..115b019e 100644 --- a/src/helpers/featureLabel.ts +++ b/src/helpers/featureLabel.ts @@ -16,7 +16,8 @@ export const getTypeLabel = ({ layer, osmMeta, properties, schema }: Feature) => const getRefLabel = (feature: Feature) => feature.tags.ref ? `${getTypeLabel(feature)} ${feature.tags.ref}` : ''; -const getName = ({ tags }: Feature) => tags[`name:${intl.lang}`] || tags.name; +export const getName = ({ tags }: Feature) => + tags[`name:${intl.lang}`] || tags.name; export const hasName = (feature: Feature) => feature.point || getName(feature) || getBuiltAddress(feature); // we dont want to show "No name" for point @@ -24,7 +25,7 @@ export const hasName = (feature: Feature) => export const getHumanPoiType = (feature: Feature) => hasName(feature) ? getTypeLabel(feature) : t('featurepanel.no_name'); -const getLabelWithoutFallback = (feature: Feature) => { +export const getLabelWithoutFallback = (feature: Feature) => { const { point, roundedCenter } = feature; if (point) { return roundedToDeg(roundedCenter); diff --git a/src/server/upload/__tests__/getUploadData.test.ts b/src/server/upload/__tests__/getUploadData.test.ts new file mode 100644 index 00000000..0d97ba9c --- /dev/null +++ b/src/server/upload/__tests__/getUploadData.test.ts @@ -0,0 +1,88 @@ +import type { Feature } from '../../../services/types'; + +import { getUploadData } from '../getUploadData'; +import { File } from '../types'; + +const feature: Feature = { + type: 'Feature', + center: [14.5255208, 50.5776264], + osmMeta: { + type: 'way', + id: 255654888, + }, + tags: { + amenity: 'place_of_worship', + name: 'svatý Mikuláš', + wikidata: 'Q15393875', + wikipedia: 'cs:Kostel svatého Mikuláše (Drchlava)', + }, + properties: { + class: 'place_of_worship', + subclass: 'place_of_worship', + }, + countryCode: 'cz', + schema: { + presetKey: 'amenity/place_of_worship/christian', + label: 'Christian Church', + } as unknown as Feature['schema'], + parentFeatures: [ + { + tags: { + climbing: 'whatever', + name: 'ClimbingCragName', + }, + } as unknown as Feature, + ], +}; + +const file: File = { + filepath: '/tmp/cda0f39b9e13dc5cd1ea4cb07', + filename: 'IMG_4424.jpg', + location: [14.525191666666668, 50.577450000000006], + date: new Date('2024-03-16T11:35:56.000Z'), +}; + +const user = { id: 162287, username: 'zby-cz' }; + +const lang = 'en'; + +const out = { + date: '2024-03-16T11:35:56Z', + filename: 'ClimbingCragName, svatý Mikuláš (Christian Church) - OsmAPP.jpg', + filepath: '/tmp/cda0f39b9e13dc5cd1ea4cb07', + photoLocation: [14.525191666666668, 50.577450000000006], + placeLocation: [14.5255208, 50.5776264], + text: expect.anything(), +}; + +//language=html +const outDescription = ` +=={{int:filedesc}}== +{{Information + |description = {{en|1=ClimbingCragName, svatý Mikuláš (Christian Church)}} + |date = 2024-03-16T11:35:56Z + |source = {{Own photo}} + |author = OpenStreetMap user [https://www.openstreetmap.org/user/zby-cz#id=162287 zby-cz] + |other_fields = + {{Information field + |name = {{Label|P1259|link=-|capitalization=ucfirst}} + |value = {{#property:P1259|from=M{{PAGEID}} }} [[File:OOjs UI icon edit-ltr-progressive.svg |frameless |text-top |10px |link={{fullurl:{{FULLPAGENAME}}}}#P1259|alt=Edit this on Structured Data on Commons|Edit this on Structured Data on Commons]] + }} + {{Information field + |name = {{Label|P9149|link=-|capitalization=ucfirst}} + |value = {{#property:P9149|from=M{{PAGEID}} }} [[File:OOjs UI icon edit-ltr-progressive.svg |frameless |text-top |10px |link={{fullurl:{{FULLPAGENAME}}}}#P9149|alt=Edit this on Structured Data on Commons|Edit this on Structured Data on Commons]] + }} + {{OSMLink |type=way |OSM_ID=255654888 }} + {{Information field |name= OsmAPP |value= https://osmapp.org/way/255654888 }} + }} + +=={{int:license-header}}== +{{Self|cc-by-sa-4.0|author=OpenStreetMap user [https://www.openstreetmap.org/user/zby-cz#id=162287 zby-cz]}} +{{FoP-Czech_Republic}} +`; + +test('getUploadData', () => { + const wikiapiUploadRequest = getUploadData(user, feature, file, lang, ''); + expect(wikiapiUploadRequest).toEqual(out); + expect(wikiapiUploadRequest.text.trim()).toEqual(outDescription.trim()); +}); diff --git a/src/server/upload/fetchToFile.ts b/src/server/upload/fetchToFile.ts new file mode 100644 index 00000000..23be6d0d --- /dev/null +++ b/src/server/upload/fetchToFile.ts @@ -0,0 +1,13 @@ +import { Readable } from 'stream'; +import fs from 'node:fs'; +import { finished } from 'stream/promises'; + +export const fetchToFile = async (url: string) => { + const filepath = `/tmp/${Math.random()}`; + const response = await fetch(url); + const inStream = Readable.fromWeb(response.body as unknown as any); + const outStream = fs.createWriteStream(filepath); + await finished(inStream.pipe(outStream)); + + return filepath; +}; diff --git a/src/server/upload/getExifData.ts b/src/server/upload/getExifData.ts new file mode 100644 index 00000000..63013679 --- /dev/null +++ b/src/server/upload/getExifData.ts @@ -0,0 +1,15 @@ +import exifr from 'exifr'; +import { LonLat } from '../../services/types'; + +export const getExifData = async (url: string) => { + const exif = await exifr.parse(url); + + const location = + exif?.latitude && exif?.longitude ? [exif.longitude, exif.latitude] : null; + + const date = exif?.DateTimeOriginal + ? new Date(exif.DateTimeOriginal) + : new Date(); + + return { location, date } as { location: LonLat | null; date: Date }; +}; diff --git a/src/server/upload/getUploadData.ts b/src/server/upload/getUploadData.ts new file mode 100644 index 00000000..b8214722 --- /dev/null +++ b/src/server/upload/getUploadData.ts @@ -0,0 +1,82 @@ +import { ServerOsmUser } from '../osmApiAuthServer'; +import type { Feature } from '../../services/types'; +import { File } from './types'; +import { getName } from '../../helpers/featureLabel'; +import { getFilename, getTitle } from './utils'; +import { + getFullOsmappLink, + getOsmappLink, + getUrlOsmId, +} from '../../services/helpers'; +import { PROJECT_ID } from '../../services/project'; + +const getOsmappUrls = (feature: Feature) => { + const osmappUrl = `https://osmapp.org${getOsmappLink(feature)}`; + const openclimbingUrl = `https://openclimbing.org${getOsmappLink(feature)}`; + + if (feature.tags.climbing) { + return `${osmappUrl}
${openclimbingUrl}`; + } else { + return `${osmappUrl}`; + } +}; + +export const getUploadData = ( + user: ServerOsmUser, + feature: Feature, + file: File, + lang: string, + suffix: string, +) => { + const name = getName(feature); + // const presetKey = feature.schema.presetKey; + const title = getTitle(feature, file); + const filename = getFilename(feature, file, suffix); + const osmUserUrl = `https://www.openstreetmap.org/user/${user.username}#id=${user.id}`; + const date = file.date.toISOString().replace(/\.\d+Z$/, 'Z'); + const osmappUrls = getOsmappUrls(feature); + + // TODO construct description (categories) + // TODO each file must belong to at least one category that describes its content or function + // TODO get category based on location eg + + // TODO get from list https://commons.wikimedia.org/wiki/Commons:Freedom_of_panorama#Summary_table + const fop = + feature.countryCode === 'cz' + ? '{{FoP-Czech_Republic}}' + : feature.countryCode === 'de' + ? '{{FoP-Germany}}' + : ''; + + // TODO add some overpass link https://www.wikidata.org/w/index.php?title=Template:Overpasslink&action=edit + const text = ` +=={{int:filedesc}}== +{{Information + |description = {{${lang}|1=${title}}} + |date = ${date} + |source = {{Own photo}} + |author = OpenStreetMap user [${osmUserUrl} ${user.username}] + |other_fields = + {{OSMLink |type=${feature.osmMeta.type} |OSM_ID=${feature.osmMeta.id} }} + {{Information field |name= OsmAPP |value= ${osmappUrls} }} +}} +{{Location}} +{{Object location}} + +=={{int:license-header}}== +{{Self|cc-by-sa-4.0|author=OpenStreetMap user [${osmUserUrl} ${user.username}]}} +${fop} +`; + + // TODO choose correct FOP based on country: https://commons.wikimedia.org/wiki/Category:FoP_templates + // TODO https://commons.wikimedia.org/wiki/Template:Geograph_from_structured_data + + return { + filepath: file.filepath, + filename, + text, + date, + photoLocation: file.location, + placeLocation: feature.center, + }; +}; diff --git a/src/server/upload/mediawiki/claimsHelpers.ts b/src/server/upload/mediawiki/claimsHelpers.ts new file mode 100644 index 00000000..50b90e84 --- /dev/null +++ b/src/server/upload/mediawiki/claimsHelpers.ts @@ -0,0 +1,92 @@ +import type { LonLat } from '../../../services/types'; + +const locationFactory = + (property: string) => + ([longitude, latitude]: LonLat) => ({ + type: 'statement', + mainsnak: { + snaktype: 'value', + property, + datavalue: { + type: 'globecoordinate', + value: { + latitude, + longitude, + globe: 'http://www.wikidata.org/entity/Q2', + precision: 0.000001, + }, + }, + }, + }); + +const createDate = (time: string) => ({ + type: 'statement', + mainsnak: { + snaktype: 'value', + property: 'P571', + datavalue: { + type: 'time', // https://www.wikidata.org/wiki/Help:Dates + value: { + time: `+${time.split('T')[0]}T00:00:00Z`, + timezone: 0, + before: 0, + after: 0, + precision: 11, // wikidata can't do hours, minutes, seconds + calendarmodel: 'http://www.wikidata.org/entity/Q1985727', + }, + }, + }, +}); + +const createCopyrightLicense = () => ({ + type: 'statement', + mainsnak: { + snaktype: 'value', + property: 'P275', + datavalue: { + type: 'wikibase-entityid', + value: { + 'entity-type': 'item', + 'numeric-id': 18199165, + id: 'Q18199165', // CC BY-SA 4.0 + }, + }, + }, +}); + +const createCopyrightStatus = () => ({ + type: 'statement', + mainsnak: { + snaktype: 'value', + property: 'P6216', + datavalue: { + type: 'wikibase-entityid', + value: { + 'entity-type': 'item', + 'numeric-id': 50423863, + id: 'Q50423863', // copyrighted + }, + }, + }, +}); + +// Entity-statement-claim-property-value (+qualifiers) +// Statement usually has one claim. Property is eg P275. +// SNAK (some notation about knowledge) is basicly a claim +// https://www.mediawiki.org/wiki/Wikibase/API, https://www.mediawiki.org/wiki/Wikibase/DataModel#Snaks +export const claimsHelpers = { + /** https://www.wikidata.org/wiki/Property:P275 copyright license */ + createCopyrightLicense, + + /** https://www.wikidata.org/wiki/Property:P6216 copyright status */ + createCopyrightStatus, + + /** https://www.wikidata.org/wiki/Property:P571 inception */ + createDate, + + /** https://www.wikidata.org/wiki/Property:P1259 point of view */ + createPhotoLocation: locationFactory('P1259'), + + /** https://www.wikidata.org/wiki/Property:P9149 coordinates of depicted place */ + createPlaceLocation: locationFactory('P9149'), +}; diff --git a/src/server/upload/mediawiki/getPageId.ts b/src/server/upload/mediawiki/getPageId.ts new file mode 100644 index 00000000..d4c0935d --- /dev/null +++ b/src/server/upload/mediawiki/getPageId.ts @@ -0,0 +1,19 @@ +import fetch from 'isomorphic-unfetch'; +import { FORMAT, WIKI_URL } from './utils'; + +/* +[ + { + "pageid": 151835618, + "ns": 6, + "title": "File:Patník N.73 (Boundary Stone) - OsmAPP.jpg" + } +] +*/ + +export const getPageId = async (title: string): Promise => { + const params = { action: 'query', titles: title, ...FORMAT }; + const response = await fetch(`${WIKI_URL}?${new URLSearchParams(params)}`); + const data = await response.json(); + return data.query.pages?.[0]?.pageid; +}; diff --git a/src/server/upload/mediawiki/getPageRevisions.ts b/src/server/upload/mediawiki/getPageRevisions.ts new file mode 100644 index 00000000..10274ad4 --- /dev/null +++ b/src/server/upload/mediawiki/getPageRevisions.ts @@ -0,0 +1,34 @@ +import fetch from 'isomorphic-unfetch'; +import { WIKI_URL } from './utils'; + +type PageInfo = { + ns: number; + title: string; + missing?: boolean; + pageid?: number; + revisions?: Revision[]; +}; + +type Revision = { + revid: number; + parentid: number; + slots: Record< + 'main' | 'mediainfo', + { contentmodel: string; contentformat: string; content: string } + >; +}; + +export const getPageRevisions = async (titles: string[]) => { + const queryString = new URLSearchParams({ + action: 'query', + prop: 'revisions', + titles: titles.join('|'), + rvprop: 'ids|content', + rvslots: '*', + formatversion: '2', + format: 'json', + }); + const response = await fetch(`${WIKI_URL}?${queryString}`); + const data = await response.json(); + return data.query.pages as PageInfo[]; +}; diff --git a/src/server/upload/mediawiki/isTitleAvailable.ts b/src/server/upload/mediawiki/isTitleAvailable.ts new file mode 100644 index 00000000..3bc1c41f --- /dev/null +++ b/src/server/upload/mediawiki/isTitleAvailable.ts @@ -0,0 +1,11 @@ +import fetch from 'isomorphic-unfetch'; +import { FORMAT, WIKI_URL } from './utils'; + +export const isTitleAvailable = async (title: string): Promise => { + const response = await fetch(WIKI_URL, { + method: 'POST', + body: new URLSearchParams({ action: 'query', titles: title, ...FORMAT }), + }); + const data = await response.json(); + return Boolean(data.query.pages[0].missing); +}; diff --git a/src/server/upload/mediawiki/mediawiki.ts b/src/server/upload/mediawiki/mediawiki.ts new file mode 100644 index 00000000..bf2affff --- /dev/null +++ b/src/server/upload/mediawiki/mediawiki.ts @@ -0,0 +1,93 @@ +import fetch from 'isomorphic-unfetch'; +import { + cookieJar, + FORMAT, + getUploadBody, + UploadParams, + WIKI_URL, +} from './utils'; +import { readFile } from 'node:fs/promises'; + +type Params = Record; + +export const getMediaWikiSession = () => { + let sessionCookie: string | undefined; + + const GET = async (action: string, params: Params) => { + const query = new URLSearchParams({ action, ...params, ...FORMAT }); + const response = await fetch(`${WIKI_URL}?${query}`, { + headers: { Cookie: sessionCookie }, + }); + sessionCookie = cookieJar(sessionCookie, response); + return response.json(); + }; + + const POST = async (action: string, params: Params) => { + const query = new URLSearchParams({ action, ...params, ...FORMAT }); + const response = await fetch(WIKI_URL, { + method: 'POST', + headers: { Cookie: sessionCookie }, + body: query, + }); + sessionCookie = cookieJar(sessionCookie, response); + return response.json(); + }; + + const UPLOAD = async (action: string, params: UploadParams) => { + const response = await fetch(WIKI_URL, { + method: 'POST', + headers: { Cookie: sessionCookie }, + body: getUploadBody({ action, ...params, ...FORMAT }), + }); + sessionCookie = cookieJar(sessionCookie, response); + return response.json(); + }; + + const getLoginToken = async (): Promise => { + const data = await GET('query', { meta: 'tokens', type: 'login' }); + return data.query.tokens.logintoken; + }; + + const login = async (lgname: string, lgpassword: string) => { + const lgtoken = await getLoginToken(); + const data = await POST('login', { lgname, lgpassword, lgtoken }); + return data.login; + }; + + const getCsrfToken = async () => { + const data = await GET('query', { meta: 'tokens', type: 'csrf' }); + return data.query.tokens.csrftoken; + }; + + const upload = async (filepath: string, filename: string, text: string) => { + const token = await getCsrfToken(); + const file = await readFile(filepath); + const blob = new Blob([file], { type: 'application/octet-stream' }); // TODO make it stream (?) + + const data = await UPLOAD('upload', { + file: blob, + filename, + text, + comment: 'Initial upload from OsmAPP.org', + token, + }); + return data; + }; + + // https://github.com/multichill/toollabs/blob/master/bot/commons/geograph_uploader.py#L132 + const editClaims = async (pageId: string, claims) => { + const token = await getCsrfToken(); + const data = await POST('wbeditentity', { + id: pageId, + data: JSON.stringify({ claims }), + token, + }); + return data; + }; + + return { + login, + upload, + editClaims, + }; +}; diff --git a/src/server/upload/mediawiki/run-cli.ts b/src/server/upload/mediawiki/run-cli.ts new file mode 100644 index 00000000..23681d80 --- /dev/null +++ b/src/server/upload/mediawiki/run-cli.ts @@ -0,0 +1,17 @@ +import { getMediaWikiSession } from './mediawiki'; +import { readFile } from 'node:fs/promises'; +import { isTitleAvailable } from './isTitleAvailable'; + +// run with: +// npx tsx run-cli.ts + +(async () => { + const password = (await readFile('../../../../.env.local', 'utf-8')) + .split('\n')[0] + .split('=')[1]; + + const session = getMediaWikiSession(); + console.log(await session.login('OsmappBot@osmapp-upload', password)); // eslint-disable-line no-console + + isTitleAvailable('File:Hrana_(Climbing_crag)_-_OsmAPP.jpg').then(console.log); // eslint-disable-line no-console +})(); diff --git a/src/server/upload/mediawiki/utils.ts b/src/server/upload/mediawiki/utils.ts new file mode 100644 index 00000000..04c5cb1f --- /dev/null +++ b/src/server/upload/mediawiki/utils.ts @@ -0,0 +1,32 @@ +export const WIKI_URL = 'https://commons.wikimedia.org/w/api.php'; +export const FORMAT = { format: 'json', formatversion: '2' }; + +export type UploadParams = Record; + +export const getUploadBody = (params: UploadParams) => { + const formData = new FormData(); + Object.entries(params).forEach(([k, v]) => { + formData.append(k, v); + }); + return formData; +}; + +export const cookieJar = (cookies: string, response: Response) => { + const oldCookies: Record = + cookies?.split(';').reduce((acc, c) => { + const [name, value] = c.split('='); + return { ...acc, [name.trim()]: value.trim() }; + }, {}) ?? {}; + + const headers = new Headers(response.headers); + headers.getSetCookie().forEach((cookie) => { + const [name, value] = cookie.split(';')[0].split('='); + oldCookies[name] = value; + }); + + const out = Object.entries(oldCookies) + .map(([name, value]) => `${name}=${value}`) + .join('; '); + + return out; +}; diff --git a/src/server/upload/types.ts b/src/server/upload/types.ts new file mode 100644 index 00000000..87e8b502 --- /dev/null +++ b/src/server/upload/types.ts @@ -0,0 +1,8 @@ +import type { LonLat } from '../../services/types'; + +export type File = { + filepath: string; + filename: string; + location: LonLat | null; + date: Date; +}; diff --git a/src/server/upload/uploadToWikimediaCommons.ts b/src/server/upload/uploadToWikimediaCommons.ts new file mode 100644 index 00000000..d8ff1978 --- /dev/null +++ b/src/server/upload/uploadToWikimediaCommons.ts @@ -0,0 +1,65 @@ +import { ServerOsmUser } from '../osmApiAuthServer'; +import type { Feature } from '../../services/types'; +import { File } from './types'; +import { getMediaWikiSession } from './mediawiki/mediawiki'; +import { findFreeSuffix } from './utils'; +import { getUploadData } from './getUploadData'; +import { getPageId } from './mediawiki/getPageId'; +import { claimsHelpers } from './mediawiki/claimsHelpers'; + +export const uploadToWikimediaCommons = async ( + user: ServerOsmUser, + feature: Feature, + file: File, + lang: string, +) => { + const password = process.env.OSMAPPBOT_PASSWORD; + if (!password) { + throw new Error('OSMAPPBOT_PASSWORD not set'); + } + + const session = getMediaWikiSession(); + await session.login('OsmappBot@osmapp-upload', password); // https://www.mediawiki.org/wiki/Special:BotPasswords + // TODO use oauth bearer + no cookie jar + + const suffix = await findFreeSuffix(feature, file); + const data = getUploadData(user, feature, file, lang, suffix); + + const uploadResult = await session.upload( + data.filepath, + data.filename, + data.text, + ); + if (uploadResult?.upload?.result !== 'Success') { + throw new Error(`Upload failed: ${JSON.stringify(uploadResult)}`); + } + + const title = `File:${uploadResult?.upload.filename}`; + const pageId = await getPageId(title); + if (!pageId) { + throw new Error( + `Page not found: ${title}, uploadResult: ${JSON.stringify(uploadResult)}`, + ); + } + + const claims = [ + claimsHelpers.createCopyrightLicense(), + claimsHelpers.createCopyrightStatus(), + claimsHelpers.createDate(data.date), + claimsHelpers.createPlaceLocation(data.placeLocation), + claimsHelpers.createPhotoLocation(data.photoLocation), + ]; + const claimsResult = await session.editClaims(`M${pageId}`, claims); + if (claimsResult.success !== 1) { + throw new Error(`Claims failed: ${JSON.stringify(claimsResult)}`); + } + + return { + title, + uploadResult, + claimsResult, + }; + + // TODO check duplicate by sha1 before upload + // MD5 hash wikidata https://commons.wikimedia.org/w/index.php?title=File%3AArea_needs_fixing-Syria_map.png&diff=801153548&oldid=607140167 +}; diff --git a/src/server/upload/utils.ts b/src/server/upload/utils.ts new file mode 100644 index 00000000..65cced52 --- /dev/null +++ b/src/server/upload/utils.ts @@ -0,0 +1,46 @@ +import { + getLabel, + getLabelWithoutFallback, + getName, + getParentLabel, + getTypeLabel, +} from '../../helpers/featureLabel'; +import type { Feature } from '../../services/types'; +import { File } from './types'; + +import { isTitleAvailable } from './mediawiki/isTitleAvailable'; +import { join } from '../../utils'; + +export const getTitle = (feature: Feature, file: File) => { + const name = join( + getParentLabel(feature), // fetched only for climbing route + ', ', + getLabelWithoutFallback(feature), + ); + const presetName = getTypeLabel(feature); + const location = file.location ?? feature.center; + return name + ? `${name} (${presetName})` + : `${presetName} at ${location.map((x) => x.toFixed(5))}`; +}; + +export const getFilename = (feature: Feature, file: File, suffix: string) => { + const title = getTitle(feature, file); + + const extension = file.filename.split('.').pop(); + const fixedExtension = extension.toLowerCase() === 'heic' ? 'jpg' : extension; + return `${title} - OsmAPP${suffix}.${fixedExtension}`; +}; + +export async function findFreeSuffix(feature: Feature, file: File) { + for (let i = 1; i < 20; i++) { + const suffix = i === 1 ? '' : ` (${i})`; + const filename = getFilename(feature, file, suffix); + const isFree = await isTitleAvailable(`File:${filename}`); + if (isFree) { + return suffix; + } + } + + throw new Error(`Could not find 20 free suffixes for ${file.filename}`); +} diff --git a/src/services/osmApi.ts b/src/services/osmApi.ts index 9d285537..013f2057 100644 --- a/src/services/osmApi.ts +++ b/src/services/osmApi.ts @@ -126,7 +126,7 @@ const getCountryCode = async (feature: Feature): Promise => { return null; }; -const getRelationElementsAndCenter = async (apiId: OsmId) => { +export const getRelationElementsAndCenter = async (apiId: OsmId) => { const element = await getOsmPromise(apiId); const getPositionOfFirstItem = isPublictransportRoute({ tags: element.tags }) || @@ -169,7 +169,7 @@ const getElementsAndCenter = async (apiId: OsmId) => { } }; -const fetchFeatureWithCenter = async (apiId: OsmId) => { +export const fetchFeatureWithCenter = async (apiId: OsmId) => { const [{ element, center }] = await Promise.all([ getElementsAndCenter(apiId), fetchSchemaTranslations(), diff --git a/yarn.lock b/yarn.lock index c1bdc4f3..822d54ed 100644 --- a/yarn.lock +++ b/yarn.lock @@ -3907,6 +3907,11 @@ execa@~8.0.1: signal-exit "^4.1.0" strip-final-newline "^3.0.0" +exifr@^7.1.3: + version "7.1.3" + resolved "https://registry.yarnpkg.com/exifr/-/exifr-7.1.3.tgz#f6218012c36dbb7d843222011b27f065fddbab6f" + integrity sha512-g/aje2noHivrRSLbAUtBPWFbxKdKhgj/xr1vATDdUXPOFYJlQ62Ft0oy+72V6XLIpDJfHs6gXLbBLAolqOXYRw== + exit@^0.1.2: version "0.1.2" resolved "https://registry.yarnpkg.com/exit/-/exit-0.1.2.tgz#0632638f8d877cc82107d30a0fff1a17cba1cd0c"