From 12637456f0fcebdfbc4b70aaae09611a0444dfe1 Mon Sep 17 00:00:00 2001 From: Andrea Ghensi Date: Mon, 5 Feb 2024 22:15:25 +0000 Subject: [PATCH] feat: handle jekyll code blocks fixes #67 --- android/app/build.gradle | 17 ++--- android/build.gradle | 13 ---- .../gradle/wrapper/gradle-wrapper.properties | 6 +- android/settings.gradle | 31 ++++++-- lib/converter.dart | 41 ++++++++-- pubspec.lock | 76 ++++++++++++------- pubspec.yaml | 2 +- test/converter_test.dart | 48 +++++++----- .../jekyll-code-block-no-language.html | 17 +++++ .../jekyll-code-block-no-language.md | 15 ++++ test/resources/jekyll-code-block.html | 17 +++++ test/resources/jekyll-code-block.md | 15 ++++ test/resources/simple.html | 2 + test/resources/simple.md | 3 + 14 files changed, 215 insertions(+), 88 deletions(-) mode change 100644 => 100755 android/app/build.gradle mode change 100644 => 100755 android/build.gradle mode change 100644 => 100755 android/gradle/wrapper/gradle-wrapper.properties mode change 100644 => 100755 android/settings.gradle mode change 100644 => 100755 lib/converter.dart mode change 100644 => 100755 test/converter_test.dart create mode 100755 test/resources/jekyll-code-block-no-language.html create mode 100755 test/resources/jekyll-code-block-no-language.md create mode 100755 test/resources/jekyll-code-block.html create mode 100755 test/resources/jekyll-code-block.md create mode 100644 test/resources/simple.html create mode 100644 test/resources/simple.md diff --git a/android/app/build.gradle b/android/app/build.gradle old mode 100644 new mode 100755 index ca99543..37e6093 --- a/android/app/build.gradle +++ b/android/app/build.gradle @@ -1,3 +1,9 @@ +plugins { + id "com.android.application" + id "kotlin-android" + id "dev.flutter.flutter-gradle-plugin" +} + def localProperties = new Properties() def localPropertiesFile = rootProject.file('local.properties') if (localPropertiesFile.exists()) { @@ -6,11 +12,6 @@ if (localPropertiesFile.exists()) { } } -def flutterRoot = localProperties.getProperty('flutter.sdk') -if (flutterRoot == null) { - throw new GradleException("Flutter SDK not found. Define location with flutter.sdk in the local.properties file.") -} - def flutterVersionCode = localProperties.getProperty('flutter.versionCode') if (flutterVersionCode == null) { flutterVersionCode = '1' @@ -21,10 +22,6 @@ if (flutterVersionName == null) { flutterVersionName = '1.0' } -apply plugin: 'com.android.application' -apply plugin: 'kotlin-android' -apply from: "$flutterRoot/packages/flutter_tools/gradle/flutter.gradle" - android { compileSdkVersion 33 @@ -63,6 +60,6 @@ flutter { } dependencies { - implementation "org.jetbrains.kotlin:kotlin-stdlib-jdk7:$kotlin_version" + implementation "org.jetbrains.kotlin:kotlin-stdlib-jdk7:1.7.20" implementation "net.dankito.readability4j:readability4j:1.0.8" } diff --git a/android/build.gradle b/android/build.gradle old mode 100644 new mode 100755 index 38de695..bc157bd --- a/android/build.gradle +++ b/android/build.gradle @@ -1,16 +1,3 @@ -buildscript { - ext.kotlin_version = '1.7.20' - repositories { - google() - mavenCentral() - } - - dependencies { - classpath 'com.android.tools.build:gradle:7.0.4' - classpath "org.jetbrains.kotlin:kotlin-gradle-plugin:$kotlin_version" - } -} - allprojects { repositories { google() diff --git a/android/gradle/wrapper/gradle-wrapper.properties b/android/gradle/wrapper/gradle-wrapper.properties old mode 100644 new mode 100755 index 595fb86..2d78c30 --- a/android/gradle/wrapper/gradle-wrapper.properties +++ b/android/gradle/wrapper/gradle-wrapper.properties @@ -1,6 +1,6 @@ -#Fri Jun 23 08:50:38 CEST 2017 +#Mon Feb 05 20:19:47 UTC 2024 distributionBase=GRADLE_USER_HOME +distributionUrl=https\://services.gradle.org/distributions/gradle-8.1.1-bin.zip distributionPath=wrapper/dists -zipStoreBase=GRADLE_USER_HOME zipStorePath=wrapper/dists -distributionUrl=https\://services.gradle.org/distributions/gradle-7.2-all.zip +zipStoreBase=GRADLE_USER_HOME diff --git a/android/settings.gradle b/android/settings.gradle old mode 100644 new mode 100755 index 44e62bc..76b0378 --- a/android/settings.gradle +++ b/android/settings.gradle @@ -1,11 +1,26 @@ -include ':app' +pluginManagement { + def flutterSdkPath = { + def properties = new Properties() + file("local.properties").withInputStream { properties.load(it) } + def flutterSdkPath = properties.getProperty("flutter.sdk") + assert flutterSdkPath != null, "flutter.sdk not set in local.properties" + return flutterSdkPath + } + settings.ext.flutterSdkPath = flutterSdkPath() -def localPropertiesFile = new File(rootProject.projectDir, "local.properties") -def properties = new Properties() + includeBuild("${settings.ext.flutterSdkPath}/packages/flutter_tools/gradle") -assert localPropertiesFile.exists() -localPropertiesFile.withReader("UTF-8") { reader -> properties.load(reader) } + repositories { + google() + mavenCentral() + gradlePluginPortal() + } +} -def flutterSdkPath = properties.getProperty("flutter.sdk") -assert flutterSdkPath != null, "flutter.sdk not set in local.properties" -apply from: "$flutterSdkPath/packages/flutter_tools/gradle/app_plugin_loader.gradle" +plugins { + id "dev.flutter.flutter-plugin-loader" version "1.0.0" + id "com.android.application" version "7.4.2" apply false + id "org.jetbrains.kotlin.android" version "1.7.20" apply false +} + +include ":app" diff --git a/lib/converter.dart b/lib/converter.dart old mode 100644 new mode 100755 index 60b4b61..6b7358f --- a/lib/converter.dart +++ b/lib/converter.dart @@ -53,16 +53,15 @@ class Url2MdConverter { "hr": "---", "bulletListMarker": "-", "codeBlockStyle": "fenced", - }); - var title = readableResults.title; - var author = readableResults.author; - var excerpt = readableResults.excerpt; + }, rules: [ + jeckyllRule + ]); return MarkdownArticle( url: url, content: markdown, - title: title, - author: author, - excerpt: excerpt, + title: readableResults.title, + author: readableResults.author, + excerpt: readableResults.excerpt, creationDate: formattedDate); } catch (e) { _notificationService.showToast("$e"); @@ -75,4 +74,32 @@ class Url2MdConverter { creationDate: formattedDate); } } + + html2md.Rule jeckyllRule = html2md.Rule('jekyll-codeblocks', + filterFn: (node) => node.nodeName == 'code' && node.parentElName == 'pre', + replacement: (content, node) { + var language = getLanguage(node); + var content = node.childNodes().map((e) => e.textContent).join(); + return '\n\n```$language\n$content\n```\n\n'; + }); +} + +String getLanguage(node) { + var regex = RegExp(r'language-(\S+)'); + var className = node.firstChild!.className; + var languageMatched = regex.firstMatch(className)?.group(1); + if (languageMatched != null) { + return languageMatched; + } + var nodeElement = node.asElement(); + while (nodeElement.parent != null) { + nodeElement = nodeElement.parent; + for (var className in nodeElement.classes) { + languageMatched = regex.firstMatch(className)?.group(1); + if (languageMatched != null) { + return languageMatched; + } + } + } + return ''; } diff --git a/pubspec.lock b/pubspec.lock index 57080d2..db01b35 100644 --- a/pubspec.lock +++ b/pubspec.lock @@ -149,10 +149,10 @@ packages: dependency: transitive description: name: collection - sha256: f092b211a4319e98e5ff58223576de6c2803db36221657b46c82574721240687 + sha256: ee67cb0715911d28db6bf4af1026078bd6f0128b07a5f66fb2ed94ec6783c09a url: "https://pub.dev" source: hosted - version: "1.17.2" + version: "1.18.0" convert: dependency: transitive description: @@ -400,6 +400,30 @@ packages: url: "https://pub.dev" source: hosted version: "4.8.1" + leak_tracker: + dependency: transitive + description: + name: leak_tracker + sha256: cdd14e3836065a1f6302a236ec8b5f700695c803c57ae11a1c84df31e6bcf831 + url: "https://pub.dev" + source: hosted + version: "10.0.3" + leak_tracker_flutter_testing: + dependency: transitive + description: + name: leak_tracker_flutter_testing + sha256: "9b2ef90589911d665277464e0482b209d39882dffaaf4ef69a3561a3354b2ebc" + url: "https://pub.dev" + source: hosted + version: "3.0.2" + leak_tracker_testing: + dependency: transitive + description: + name: leak_tracker_testing + sha256: fd3cd66cb2bcd7b50dcd3b413af49d78051f809c8b3f6e047962765c15a0d23d + url: "https://pub.dev" + source: hosted + version: "3.0.0" lints: dependency: transitive description: @@ -428,26 +452,26 @@ packages: dependency: transitive description: name: matcher - sha256: "1803e76e6653768d64ed8ff2e1e67bea3ad4b923eb5c56a295c3e634bad5960e" + sha256: d2323aa2060500f906aa31a895b4030b6da3ebdcc5619d14ce1aada65cd161cb url: "https://pub.dev" source: hosted - version: "0.12.16" + version: "0.12.16+1" material_color_utilities: dependency: transitive description: name: material_color_utilities - sha256: "9528f2f296073ff54cb9fee677df673ace1218163c3bc7628093e7eed5203d41" + sha256: "0e0a020085b65b6083975e499759762399b4475f766c21668c4ecca34ea74e5a" url: "https://pub.dev" source: hosted - version: "0.5.0" + version: "0.8.0" meta: dependency: transitive description: name: meta - sha256: "3c74dbf8763d36539f114c799d8a2d87343b5067e9d796ca22b5eb8437090ee3" + sha256: d584fa6707a52763a52446f02cc621b077888fb63b93bbcb1143a7be5a0c0c04 url: "https://pub.dev" source: hosted - version: "1.9.1" + version: "1.11.0" mime: dependency: transitive description: @@ -476,10 +500,10 @@ packages: dependency: transitive description: name: path - sha256: "8829d8a55c13fc0e37127c29fedf290c102f4e40ae94ada574091fe0ff96c917" + sha256: "087ce49c3f0dc39180befefc60fdb4acd8f8620e5682fe2476afd0b3688bb4af" url: "https://pub.dev" source: hosted - version: "1.8.3" + version: "1.9.0" path_provider: dependency: transitive description: @@ -698,18 +722,18 @@ packages: dependency: transitive description: name: stack_trace - sha256: c3c7d8edb15bee7f0f74debd4b9c5f3c2ea86766fe4178eb2a18eb30a0bdaed5 + sha256: "73713990125a6d93122541237550ee3352a2d84baad52d375a4cad2eb9b7ce0b" url: "https://pub.dev" source: hosted - version: "1.11.0" + version: "1.11.1" stream_channel: dependency: transitive description: name: stream_channel - sha256: "83615bee9045c1d322bbbd1ba209b7a749c2cbcdcb3fdd1df8eb488b3279c1c8" + sha256: ba2aa5d8cc609d96bbb2899c28934f9e1af5cddbd60a827822ea467161eb54e7 url: "https://pub.dev" source: hosted - version: "2.1.1" + version: "2.1.2" stream_transform: dependency: transitive description: @@ -738,10 +762,10 @@ packages: dependency: transitive description: name: test_api - sha256: "75760ffd7786fffdfb9597c35c5b27eaeec82be8edfb6d71d32651128ed7aab8" + sha256: "5c2f730018264d276c20e4f1503fd1308dfbbae39ec8ee63c5236311ac06954b" url: "https://pub.dev" source: hosted - version: "0.6.0" + version: "0.6.1" timing: dependency: transitive description: @@ -806,22 +830,22 @@ packages: url: "https://pub.dev" source: hosted version: "2.1.4" - watcher: + vm_service: dependency: transitive description: - name: watcher - sha256: "3d2ad6751b3c16cf07c7fca317a1413b3f26530319181b37e3b9039b84fc01d8" + name: vm_service + sha256: b3d56ff4341b8f182b96aceb2fa20e3dcb336b9f867bc0eafc0de10f1048e957 url: "https://pub.dev" source: hosted - version: "1.1.0" - web: + version: "13.0.0" + watcher: dependency: transitive description: - name: web - sha256: dc8ccd225a2005c1be616fe02951e2e342092edf968cf0844220383757ef8f10 + name: watcher + sha256: "3d2ad6751b3c16cf07c7fca317a1413b3f26530319181b37e3b9039b84fc01d8" url: "https://pub.dev" source: hosted - version: "0.1.4-beta" + version: "1.1.0" web_socket_channel: dependency: transitive description: @@ -855,5 +879,5 @@ packages: source: hosted version: "3.1.2" sdks: - dart: ">=3.1.0 <4.0.0" - flutter: ">=3.13.0" + dart: ">=3.2.0 <4.0.0" + flutter: ">=3.18.0-18.0.pre.54" diff --git a/pubspec.yaml b/pubspec.yaml index 64f39fa..bbe3842 100644 --- a/pubspec.yaml +++ b/pubspec.yaml @@ -9,7 +9,7 @@ dependencies: sdk: flutter cupertino_icons: ^1.0.5 http: ^1.1.0 - html2md: ^1.2.5 + html2md: ^1.3.1 intl: ^0.18.1 share_plus: ^6.3.1 receive_sharing_intent: diff --git a/test/converter_test.dart b/test/converter_test.dart old mode 100644 new mode 100755 index 0a06d42..c0658b2 --- a/test/converter_test.dart +++ b/test/converter_test.dart @@ -1,3 +1,4 @@ +import 'dart:io'; import 'package:flutter_test/flutter_test.dart'; import 'package:markdownr/converter.dart'; import 'package:markdownr/httpclient.dart'; @@ -14,6 +15,8 @@ import 'package:mockito/mockito.dart'; import 'converter_test.mocks.dart'; void main() { + TestWidgetsFlutterBinding.ensureInitialized(); + group('Url2MdConverter', () { late MockHttpClient mockHttpClient; late MockNotificationService mockNotificationService; @@ -31,26 +34,6 @@ void main() { ); }); - test('convert should return valid markdown', () async { - const url = 'https://example.com'; - const content = '

Test Header

Test paragraph

'; - const html = - 'Test Title$content'; - when(mockHttpClient.getPage(url)).thenAnswer((_) async => html); - when(mockReadabilityService.makeReadable(html, url)).thenAnswer( - (_) async => const ReadabilityOutput( - content: '
$content
', - title: "Test Title", - author: "Myself", - excerpt: "A really good article")); - final markdown = await url2mdConverter.convertPage(url: url); - expect(markdown.url, url); - expect(markdown.content, "## Test Header\n\nTest paragraph"); - expect(markdown.title, "Test Title"); - expect(markdown.author, "Myself"); - expect(markdown.excerpt, "A really good article"); - }); - test('convert should handle exceptions', () async { const url = 'https://example.com'; when(mockHttpClient.getPage(url)).thenThrow(Exception()); @@ -61,5 +44,30 @@ void main() { expect(markdown.author, ""); expect(markdown.excerpt, ""); }); + + void fileBasedTestCase(fileName) { + test('Convert case $fileName', () async { + const url = "https://www.example.com"; + final file = File('test/resources/$fileName.html'); + var content = await file.readAsString(); + final html = + 'Title$content'; + when(mockHttpClient.getPage(url)).thenAnswer((_) async => html); + when(mockReadabilityService.makeReadable(html, url)).thenAnswer( + (_) async => ReadabilityOutput( + content: content, + title: "Title", + author: "Author", + excerpt: "A really good article")); + final markdown = await url2mdConverter.convertPage(url: url); + final mdFile = File('test/resources/$fileName.md'); + var mdContent = await mdFile.readAsString(); + expect(markdown.content, mdContent); + }); + } + + fileBasedTestCase('simple'); + fileBasedTestCase('jekyll-code-block'); + fileBasedTestCase('jekyll-code-block-no-language'); }); } diff --git a/test/resources/jekyll-code-block-no-language.html b/test/resources/jekyll-code-block-no-language.html new file mode 100755 index 0000000..eccca99 --- /dev/null +++ b/test/resources/jekyll-code-block-no-language.html @@ -0,0 +1,17 @@ +
+
+
#! /usr/bin/env python3
+
+import tika
+from tika import parser
+
+fileIn = "berk011veel01_01.epub"
+fileOut = "berk011veel01_01.txt"
+
+parsed = parser.from_file(fileIn)
+content = parsed["content"]
+
+with open(fileOut, 'w', encoding='utf-8') as fout:
+    fout.write(content)
+
+
diff --git a/test/resources/jekyll-code-block-no-language.md b/test/resources/jekyll-code-block-no-language.md new file mode 100755 index 0000000..3d7bda2 --- /dev/null +++ b/test/resources/jekyll-code-block-no-language.md @@ -0,0 +1,15 @@ +``` +#! /usr/bin/env python3 + +import tika +from tika import parser + +fileIn = "berk011veel01_01.epub" +fileOut = "berk011veel01_01.txt" + +parsed = parser.from_file(fileIn) +content = parsed["content"] + +with open(fileOut, 'w', encoding='utf-8') as fout: + fout.write(content) +``` \ No newline at end of file diff --git a/test/resources/jekyll-code-block.html b/test/resources/jekyll-code-block.html new file mode 100755 index 0000000..dbf2aea --- /dev/null +++ b/test/resources/jekyll-code-block.html @@ -0,0 +1,17 @@ +
+
+
#! /usr/bin/env python3
+
+import tika
+from tika import parser
+
+fileIn = "berk011veel01_01.epub"
+fileOut = "berk011veel01_01.txt"
+
+parsed = parser.from_file(fileIn)
+content = parsed["content"]
+
+with open(fileOut, 'w', encoding='utf-8') as fout:
+    fout.write(content)
+
+
diff --git a/test/resources/jekyll-code-block.md b/test/resources/jekyll-code-block.md new file mode 100755 index 0000000..0d5ac22 --- /dev/null +++ b/test/resources/jekyll-code-block.md @@ -0,0 +1,15 @@ +```python +#! /usr/bin/env python3 + +import tika +from tika import parser + +fileIn = "berk011veel01_01.epub" +fileOut = "berk011veel01_01.txt" + +parsed = parser.from_file(fileIn) +content = parsed["content"] + +with open(fileOut, 'w', encoding='utf-8') as fout: + fout.write(content) +``` \ No newline at end of file diff --git a/test/resources/simple.html b/test/resources/simple.html new file mode 100644 index 0000000..d600789 --- /dev/null +++ b/test/resources/simple.html @@ -0,0 +1,2 @@ +

Test Header

+

Test paragraph

diff --git a/test/resources/simple.md b/test/resources/simple.md new file mode 100644 index 0000000..29e5b62 --- /dev/null +++ b/test/resources/simple.md @@ -0,0 +1,3 @@ +## Test Header + +Test paragraph \ No newline at end of file