diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
index 17cea76..ab0899f 100644
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@@ -10,7 +10,7 @@ jobs:
- uses: actions/checkout@v2
- uses: dart-lang/setup-dart@v1
- run: dart --version
- - run: pub get
+ - run: dart pub get
- run: dart analyze --fatal-warnings .
- run: dart format --set-exit-if-changed .
- run: dart run test -x integration
diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml
index 5cf3c78..a787e31 100644
--- a/.github/workflows/release.yml
+++ b/.github/workflows/release.yml
@@ -11,8 +11,8 @@ jobs:
- uses: actions/checkout@v2
- uses: dart-lang/setup-dart@v1
- run: dart --version
- - run: pub get
- - run: pub global activate grinder
+ - run: dart pub get
+ - run: dart pub global activate grinder
- run: echo "$HOME/.pub-cache/bin" >> $GITHUB_PATH
- name: Package and publish
run: grind pkg-github-all
diff --git a/.gitignore b/.gitignore
index 590b0d4..830b79e 100644
--- a/.gitignore
+++ b/.gitignore
@@ -3,7 +3,6 @@
# Files and directories created by pub
.buildlog
.dart_tool/
-.packages
.project
.pub/
build/
diff --git a/CHANGELOG.md b/CHANGELOG.md
index 9da4241..6511af5 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -1,3 +1,11 @@
+## 3.0.0-dev
+
+- Require Dart 2.18 and support Dart 3
+- Rename library to `linkcheck` instead of `linkcheck.run`
+- Update to the latest dependencies supporting sound null safety
+- Switch to Dart recommended lints (`package:lints/recommended.yaml`)
+- Use objects instead of maps to communicate between isolates
+
## 2.0.23
- Fix another issue with building artifacts through `grindr`/`cli_pkg`
diff --git a/README.md b/README.md
index 9d5314d..07508e4 100644
--- a/README.md
+++ b/README.md
@@ -330,6 +330,6 @@ building binaries and placing a new release into
[github.com/filiph/linkcheck/releases](https://github.com/filiph/linkcheck/releases).
In order to populate it to the [GitHub Actions Marketplace](https://github.com/marketplace/actions/check-links-with-linkcheck)
-as well, it's currently requiered to manually Edit and hit
+as well, it's currently required to manually Edit and hit
Update release on the release page once. No changes needed.
(Source: [GiHub Community](https://github.community/t/automatically-publish-action-to-marketplace-on-release/17978))
diff --git a/analysis_options.yaml b/analysis_options.yaml
index 833d692..df74cd6 100644
--- a/analysis_options.yaml
+++ b/analysis_options.yaml
@@ -1,11 +1,17 @@
-include: package:pedantic/analysis_options.yaml
+include: package:lints/recommended.yaml
analyzer:
- strong-mode:
- implicit-casts: false
- implicit-dynamic: false
+ language:
+ strict-casts: true
+ strict-inference: true
+ strict-raw-types: true
linter:
rules:
- omit_local_variable_types: false # TODO: remove and fix
- prefer_single_quotes: false
+ - always_declare_return_types
+ - comment_references
+ - prefer_final_fields
+ - type_annotate_public_apis
+ - unawaited_futures
+ - use_enums
+ - use_super_parameters
diff --git a/bin/linkcheck.dart b/bin/linkcheck.dart
index c52a027..1947955 100644
--- a/bin/linkcheck.dart
+++ b/bin/linkcheck.dart
@@ -1,5 +1,3 @@
-library linkcheck.executable;
-
import 'dart:async';
import 'dart:io';
diff --git a/dart_test.yaml b/dart_test.yaml
index 32531de..a308994 100644
--- a/dart_test.yaml
+++ b/dart_test.yaml
@@ -2,5 +2,5 @@ tags:
integration:
skip: >
The integration tests depend on serving the case folders, and for that
- we need to know the path to those (which pub run test doesn't support).
- Use `dart test/e2e_test.dart` to run these.
+ we need to know the path to those (which dart test doesn't support).
+ Use `dart run test/e2e_test.dart` to run these.
diff --git a/lib/linkcheck.dart b/lib/linkcheck.dart
index 3313796..aabf6ad 100644
--- a/lib/linkcheck.dart
+++ b/lib/linkcheck.dart
@@ -1,14 +1,12 @@
-library linkcheck.run;
-
import 'dart:async';
import 'dart:io' hide Link;
import 'package:args/args.dart';
import 'package:console/console.dart';
-import 'package:linkcheck/src/parsers/url_skipper.dart';
import 'src/crawl.dart' show crawl, CrawlResult;
import 'src/link.dart' show Link;
+import 'src/parsers/url_skipper.dart';
import 'src/writer_report.dart' show reportForWriters;
export 'src/crawl.dart' show crawl, CrawlResult;
@@ -28,7 +26,7 @@ const hostsFlag = "hosts";
const inputFlag = "input-file";
const redirectFlag = "show-redirects";
const skipFlag = "skip-file";
-const version = "2.0.23";
+const version = "3.0.0-dev";
const versionFlag = "version";
final _portOnlyRegExp = RegExp(r"^:\d+$");
@@ -224,8 +222,8 @@ Future run(List arguments, Stdout stdout) async {
bool shouldCheckExternal = argResults[externalFlag] == true;
bool showRedirects = argResults[redirectFlag] == true;
bool shouldCheckAnchors = argResults[anchorFlag] == true;
- String inputFile = argResults[inputFlag] as String;
- String skipFile = argResults[skipFlag] as String;
+ String? inputFile = argResults[inputFlag] as String?;
+ String? skipFile = argResults[skipFlag] as String?;
List urls = argResults.rest.toList();
UrlSkipper skipper = UrlSkipper.empty();
@@ -261,7 +259,7 @@ Future run(List arguments, Stdout stdout) async {
}
// TODO: exit gracefully if provided URL isn't a parseable URI
- List uris = urls.map((url) => Uri.parse(url)).toList();
+ List uris = urls.map((url) => Uri.parse(url)).toList(growable: false);
Set hosts;
if ((argResults[hostsFlag] as Iterable).isNotEmpty) {
hosts = Set.from(argResults[hostsFlag] as Iterable);
diff --git a/lib/src/crawl.dart b/lib/src/crawl.dart
index 8597e89..d25d999 100644
--- a/lib/src/crawl.dart
+++ b/lib/src/crawl.dart
@@ -1,18 +1,17 @@
-library linkcheck.crawl;
-
import 'dart:async';
import 'dart:collection';
import 'dart:io' show Stdout;
import 'package:console/console.dart';
+import 'package:meta/meta.dart';
import 'destination.dart';
import 'link.dart';
-import 'package:linkcheck/src/parsers/url_skipper.dart';
-import 'uri_glob.dart';
+import 'parsers/url_skipper.dart';
import 'server_info.dart';
-import 'worker/pool.dart';
+import 'uri_glob.dart';
import 'worker/fetch_results.dart';
+import 'worker/pool.dart';
/// Number of isolates to create by default.
const defaultThreads = 8;
@@ -36,8 +35,8 @@ Future crawl(
// Redirect output to injected [stdout] for better testing.
void print(Object message) => stdout.writeln(message);
- Cursor cursor;
- TextPen pen;
+ Cursor? cursor;
+ TextPen? pen;
if (ansiTerm) {
Console.init();
cursor = Cursor();
@@ -64,7 +63,9 @@ Future crawl(
..isSource = true
..isExternal = false)
.toSet());
- open.forEach((destination) => bin[destination.url] = Bin.open);
+ for (var destination in open) {
+ bin[destination.url] = Bin.open;
+ }
// Queue for the external destinations.
Queue openExternal = Queue();
@@ -106,7 +107,7 @@ Future crawl(
int count = 0;
if (!verbose) {
- if (ansiTerm) {
+ if (cursor != null) {
cursor.write("Crawling: $count");
} else {
print("Crawling...");
@@ -116,12 +117,12 @@ Future crawl(
// TODO:
// - --cache for creating a .linkcheck.cache file
- var allDone = Completer();
+ var allDone = Completer();
// Respond to Ctrl-C
- StreamSubscription stopSignalSubscription;
+ late final StreamSubscription stopSignalSubscription;
stopSignalSubscription = stopSignal.listen((dynamic _) async {
- if (ansiTerm) {
+ if (pen != null) {
pen
.text("\n")
.red()
@@ -148,11 +149,11 @@ Future crawl(
}
}
- bool _serverIsKnown(Destination destination) =>
+ bool serverIsKnown(Destination destination) =>
servers.keys.contains(destination.uri.authority);
Iterable availableDestinations =
- _zip(open.where(_serverIsKnown), openExternal.where(_serverIsKnown));
+ _zip(open.where(serverIsKnown), openExternal.where(serverIsKnown));
// In order not to touch the underlying iterables, we keep track
// of the destinations we want to remove.
@@ -164,8 +165,8 @@ Future crawl(
destinationsToRemove.add(destination);
String host = destination.uri.authority;
- ServerInfo server = servers[host];
- if (server.hasNotConnected) {
+ ServerInfo? server = servers[host];
+ if (server == null || server.hasNotConnected) {
destination.didNotConnect = true;
closed.add(destination);
bin[destination.url] = Bin.closed;
@@ -176,8 +177,9 @@ Future crawl(
continue;
}
- if (server.bouncer != null &&
- !server.bouncer.allows(destination.uri.path)) {
+ var serverBouncer = server.bouncer;
+ if (serverBouncer != null &&
+ !serverBouncer.allows(destination.uri.path)) {
destination.wasDeniedByRobotsTxt = true;
closed.add(destination);
bin[destination.url] = Bin.closed;
@@ -236,7 +238,7 @@ Future crawl(
"${result.didNotConnect ? 'didn\'t connect' : 'connected'}, "
"${result.robotsTxtContents.isEmpty ? 'no robots.txt' : 'robots.txt found'}.");
} else {
- if (ansiTerm) {
+ if (cursor != null) {
cursor.moveLeft(count.toString().length);
count += 1;
cursor.write(count.toString());
@@ -259,7 +261,7 @@ Future crawl(
if (destinations.isEmpty) {
if (verbose) {
print("WARNING: Received result for a destination that isn't in "
- "the inProgress set: ${result.toMap()}");
+ "the inProgress set: $result");
var isInOpen =
open.where((dest) => dest.url == result.checked.url).isNotEmpty;
var isInOpenExternal = openExternal
@@ -287,12 +289,12 @@ Future crawl(
if (verbose) {
count += 1;
print("Done checking: $checked (${checked.statusDescription}) "
- "=> ${result?.links?.length ?? 0} links");
+ "=> ${result.links.length} links");
if (checked.isBroken) {
print("- BROKEN");
}
} else {
- if (ansiTerm) {
+ if (cursor != null) {
cursor.moveLeft(count.toString().length);
count += 1;
cursor.write(count.toString());
@@ -436,11 +438,10 @@ Future crawl(
}
// Fix links (dedupe destinations).
- var urlMap = Map.fromIterable(closed,
- key: (Object dest) => (dest as Destination).url);
+ var urlMap = {for (final destination in closed) destination.url: destination};
for (var link in links) {
var canonical = urlMap[link.destination.url];
- // Note: If it wasn't for the posibility to SIGINT the process, we could
+ // Note: If it wasn't for the possibility to SIGINT the process, we could
// assert there is exactly one Destination per URL. There might not be,
// though.
if (canonical != null) {
@@ -472,9 +473,11 @@ Future crawl(
return CrawlResult(links, closed);
}
+@immutable
class CrawlResult {
final Set links;
final Set destinations;
+
const CrawlResult(this.links, this.destinations);
}
diff --git a/lib/src/destination.dart b/lib/src/destination.dart
index 5eb0af2..ad44587 100644
--- a/lib/src/destination.dart
+++ b/lib/src/destination.dart
@@ -1,8 +1,8 @@
-library linkcheck.destination;
-
import 'dart:io' show ContentType, HttpClientResponse, RedirectInfo;
-import 'package:linkcheck/src/parsers/html.dart';
+import 'package:meta/meta.dart';
+
+import 'parsers/html.dart';
/// RegExp for detecting URI scheme, such as `http:`, `mailto:`, etc.
final _scheme = RegExp(r"$(\w[\w\-]*\w):");
@@ -15,7 +15,7 @@ final _scheme = RegExp(r"$(\w[\w\-]*\w):");
/// component.
bool checkSchemeSupported(String url, Uri source) {
var match = _scheme.firstMatch(url);
- String scheme;
+ String? scheme;
if (match == null) {
// No scheme provided, so the source's scheme is used.
scheme = source.scheme;
@@ -26,20 +26,14 @@ bool checkSchemeSupported(String url, Uri source) {
return Destination.supportedSchemes.contains(scheme);
}
+@immutable
class BasicRedirectInfo {
- String url;
- int statusCode;
-
- BasicRedirectInfo.from(RedirectInfo info) {
- url = info.location.toString();
- statusCode = info.statusCode;
- }
-
- BasicRedirectInfo.fromMap(Map map)
- : url = map["url"] as String,
- statusCode = map["statusCode"] as int;
+ final String url;
+ final int statusCode;
- Map toMap() => {"url": url, "statusCode": statusCode};
+ BasicRedirectInfo.from(RedirectInfo info)
+ : url = info.location.toString(),
+ statusCode = info.statusCode;
}
class Destination {
@@ -49,20 +43,20 @@ class Destination {
final String url;
/// The uri as specified by source file, without the fragment.
- Uri _uri;
+ Uri? _uri;
/// The HTTP status code returned.
- int statusCode;
+ int? statusCode;
/// MimeType of the response.
- ContentType contentType;
+ ContentType? contentType;
- List redirects;
+ List redirects = [];
/// Url after all redirects.
- String finalUrl;
+ String? finalUrl;
- bool isExternal;
+ bool isExternal = true;
/// True if this [Destination] is parseable and could contain links to
/// other destinations. For example, HTML and CSS files are sources. JPEGs
@@ -72,7 +66,7 @@ class Destination {
/// Set of anchors on the page.
///
/// Only for [isSource] == `true`.
- List anchors;
+ List anchors = [];
/// If the URL is unparseable (malformed), this will be `true`.
bool isInvalid = false;
@@ -88,9 +82,7 @@ class Destination {
// them via toMap?
bool wasDeniedByRobotsTxt = false;
- int _hashCode;
-
- Uri _finalUri;
+ Uri? _finalUri;
/// The encoding is not UTF-8 or LATIN-1.
bool hasUnsupportedEncoding = false;
@@ -100,50 +92,19 @@ class Destination {
bool wasParsed = false;
- bool _isUnsupportedScheme;
+ bool? _isUnsupportedScheme;
Destination(Uri uri)
: url = uri.removeFragment().toString(),
- _uri = uri.removeFragment() {
- _hashCode = url.hashCode;
- }
-
- factory Destination.fromMap(Map map) {
- var destination = Destination.fromString(map["url"] as String);
- var contentType = map["primaryType"] == null
- ? null
- : ContentType(map["primaryType"] as String, map["subType"] as String);
- destination
- ..statusCode = map["statusCode"] as int
- ..contentType = contentType
- ..redirects = (map["redirects"] as List>)
- ?.map((obj) => BasicRedirectInfo.fromMap(obj))
- ?.toList()
- ..finalUrl = map["finalUrl"] as String
- ..isExternal = map["isExternal"] as bool
- ..isSource = map["isSource"] as bool
- ..anchors = map["anchors"] as List
- ..isInvalid = map["isInvalid"] as bool
- ..didNotConnect = map["didNotConnect"] as bool
- ..wasParsed = map["wasParsed"] as bool
- ..hasUnsupportedEncoding = map["hasUnsupportedEncoding"] as bool;
- return destination;
- }
+ _uri = uri.removeFragment();
Destination.fromString(String url)
- : url = url.contains("#") ? url.split("#").first : url {
- _hashCode = this.url.hashCode;
- }
+ : url = url.contains("#") ? url.split("#").first : url;
- Destination.invalid(String url)
- : url = url,
- isInvalid = true {
- _hashCode = url.hashCode;
- }
+ Destination.invalid(this.url) : isInvalid = true;
- Destination.unsupported(String url) : url = url {
+ Destination.unsupported(this.url) {
_isUnsupportedScheme = true;
- _hashCode = url.hashCode;
}
// TODO: make sure we don't assign the same hashcode to two destinations like
@@ -152,7 +113,7 @@ class Destination {
Uri get finalUri => _finalUri ??= Uri.parse(finalUrl ?? url);
@override
- int get hashCode => _hashCode;
+ int get hashCode => url.hashCode;
/// A bad or busted server didn't give us any content type. This is a warning.
bool get hasNoMimeType => wasTried && contentType == null;
@@ -176,15 +137,14 @@ class Destination {
bool get isParseableMimeType => isHtmlMimeType || isCssMimeType;
bool get isPermanentlyRedirected =>
- redirects != null &&
- redirects.isNotEmpty &&
- redirects.first.statusCode == 301;
+ redirects.isNotEmpty && redirects.first.statusCode == 301;
- bool get isRedirected => redirects != null && redirects.isNotEmpty;
+ bool get isRedirected => redirects.isNotEmpty;
/// True if the destination URI isn't one of the [supportedSchemes].
- bool get isUnsupportedScheme {
- if (_isUnsupportedScheme != null) return _isUnsupportedScheme;
+ late final bool isUnsupportedScheme = () {
+ var specifiedUnsupported = _isUnsupportedScheme;
+ if (specifiedUnsupported != null) return specifiedUnsupported;
bool result = true;
try {
// This can throw a FormatException when the URI cannot be parsed.
@@ -192,9 +152,8 @@ class Destination {
} on FormatException {
// Pass.
}
- _isUnsupportedScheme = result;
return result;
- }
+ }();
String get statusDescription {
if (isUnsupportedScheme) return "scheme unsupported";
@@ -212,15 +171,15 @@ class Destination {
}
Uri get uri {
- if (_uri != null) return _uri;
+ var specifiedUri = _uri;
+ if (specifiedUri != null) return specifiedUri;
try {
- _uri = Uri.parse(url);
+ return _uri = Uri.parse(url);
} on FormatException catch (e, s) {
print("Stack trace: $s");
throw StateError("Tried parsing '$url' as URI:\n"
"$e");
}
- return _uri;
}
bool get wasTried => didNotConnect || statusCode != null;
@@ -232,28 +191,11 @@ class Destination {
/// Returns `true` if the [fragment] (such as #something) will find it's mark
/// on this [Destination]. If the fragment is `null` or empty, it will
/// automatically succeed.
- bool satisfiesFragment(String fragment) {
+ bool satisfiesFragment(String? fragment) {
if (fragment == null || fragment == '') return true;
- if (anchors == null) return false;
return anchors.contains(normalizeAnchor(fragment));
}
- Map toMap() => {
- "url": url,
- "statusCode": statusCode,
- "primaryType": contentType?.primaryType,
- "subType": contentType?.subType,
- "redirects": redirects?.map((info) => info.toMap())?.toList(),
- "finalUrl": finalUrl,
- "isExternal": isExternal,
- "isSource": isSource,
- "anchors": anchors,
- "isInvalid": isInvalid,
- "didNotConnect": didNotConnect,
- "wasParsed": wasParsed,
- "hasUnsupportedEncoding": hasUnsupportedEncoding
- };
-
@override
String toString() => url;
@@ -261,9 +203,11 @@ class Destination {
assert(url == result.url);
finalUrl = result.finalUrl;
statusCode = result.statusCode;
- contentType = result.primaryType == null
+ var primaryType = result.primaryType;
+ var subType = result.subType;
+ contentType = primaryType == null || subType == null
? null
- : ContentType(result.primaryType, result.subType);
+ : ContentType(primaryType, subType);
redirects = result.redirects;
isSource = result.isSource;
anchors = result.anchors;
@@ -275,56 +219,35 @@ class Destination {
/// Data about destination coming from a fetch.
class DestinationResult {
- String url;
- String finalUrl;
- int statusCode;
- String primaryType;
- String subType;
+ final String url;
+ String? finalUrl;
+ int? statusCode;
+ String? primaryType;
+ String? subType;
List redirects;
- bool isSource = false;
+ final bool isSource;
List anchors;
- bool didNotConnect = false;
+ final bool didNotConnect;
bool wasParsed = false;
bool hasUnsupportedEncoding = false;
- DestinationResult.fromDestination(Destination destination)
+ DestinationResult.fromDestination(Destination destination,
+ {this.didNotConnect = false,
+ this.redirects = const [],
+ this.anchors = const []})
+ : url = destination.url,
+ isSource = destination.isSource;
+
+ DestinationResult.fromResponse(
+ Destination destination, HttpClientResponse response)
: url = destination.url,
isSource = destination.isSource,
- redirects = [];
-
- DestinationResult.fromMap(Map map)
- : url = map["url"] as String,
- finalUrl = map["finalUrl"] as String,
- statusCode = map["statusCode"] as int,
- primaryType = map["primaryType"] as String,
- subType = map["subType"] as String,
- redirects = (map["redirects"] as List>)
- .map((obj) => BasicRedirectInfo.fromMap(obj))
- .toList(),
- isSource = map["isSource"] as bool,
- anchors = map["anchors"] as List,
- didNotConnect = map["didNotConnect"] as bool,
- wasParsed = map["wasParsed"] as bool,
- hasUnsupportedEncoding = map["hasUnsupportedEncoding"] as bool;
-
- Map toMap() => {
- "url": url,
- "finalUrl": finalUrl,
- "statusCode": statusCode,
- "primaryType": primaryType,
- "subType": subType,
- "redirects": redirects.map((info) => info.toMap()).toList(),
- "isSource": isSource,
- "anchors": anchors,
- "didNotConnect": didNotConnect,
- "wasParsed": wasParsed,
- "hasUnsupportedEncoding": hasUnsupportedEncoding
- };
-
- void updateFromResponse(HttpClientResponse response) {
- statusCode = response.statusCode;
- redirects =
- response.redirects.map((info) => BasicRedirectInfo.from(info)).toList();
+ redirects = response.redirects
+ .map((info) => BasicRedirectInfo.from(info))
+ .toList(growable: false),
+ statusCode = response.statusCode,
+ anchors = const [],
+ didNotConnect = false {
if (redirects.isEmpty) {
finalUrl = url;
} else {
@@ -335,9 +258,10 @@ class DestinationResult {
current.resolve(redirect.url))
.toString();
}
- if (response.headers.contentType != null) {
- primaryType = response.headers.contentType.primaryType;
- subType = response.headers.contentType.subType;
+ var contentType = response.headers.contentType;
+ if (contentType != null) {
+ primaryType = contentType.primaryType;
+ subType = contentType.subType;
}
}
}
diff --git a/lib/src/link.dart b/lib/src/link.dart
index 2a9d428..cafe35a 100644
--- a/lib/src/link.dart
+++ b/lib/src/link.dart
@@ -1,12 +1,10 @@
-library linkcheck.link;
-
-import 'package:linkcheck/src/destination.dart';
-import 'package:linkcheck/src/origin.dart';
+import 'destination.dart';
+import 'origin.dart';
class Link {
- Origin origin;
+ final Origin origin;
Destination destination;
- String fragment;
+ final String? fragment;
/// Whether or not this link was marked as skipped.
///
@@ -14,17 +12,10 @@ class Link {
/// has a match, [wasSkipped] will be `true`.
bool wasSkipped = false;
- Link(this.origin, this.destination, String fragment,
+ Link(this.origin, this.destination, String? fragment,
[this.wasSkipped = false])
: fragment = fragment == null || fragment.isEmpty ? null : fragment;
- Link.fromMap(Map map)
- : this(
- Origin.fromMap(map["origin"] as Map),
- Destination.fromMap(map["destination"] as Map),
- map["destinationAnchor"] as String,
- map["wasSkipped"] as bool);
-
bool get breaksAnchor =>
!wasSkipped &&
destination.wasParsed &&
@@ -56,15 +47,8 @@ class Link {
bool hasWarning(bool shouldCheckAnchors) =>
(shouldCheckAnchors && breaksAnchor) || destination.hasNoMimeType;
- Map toMap() => {
- "origin": origin.toMap(),
- "destination": destination.toMap(),
- "destinationAnchor": fragment,
- "wasSkipped": wasSkipped
- };
-
@override
String toString() => "$origin => $destination"
- "${fragment == null ? '' : '#' + fragment} "
+ "${fragment == null ? '' : '#$fragment'} "
"(${destination.statusDescription})";
}
diff --git a/lib/src/origin.dart b/lib/src/origin.dart
index 6e0ec3f..1f18524 100644
--- a/lib/src/origin.dart
+++ b/lib/src/origin.dart
@@ -1,9 +1,9 @@
-library linkcheck.source;
-
+import 'package:meta/meta.dart';
import 'package:source_span/source_span.dart';
/// Origin of a link. Contains information about the exact place in a file
/// (URI) and some additional helpful info.
+@immutable
class Origin {
final Uri uri;
final SourceSpan span;
@@ -13,47 +13,6 @@ class Origin {
Origin(this.uri, this.span, this.tagName, this.text, this.outerHtml);
- Origin.fromMap(Map map)
- : this(
- Uri.parse(map["uri"] as String),
- _deserializeSpan(map["span"] as Map),
- map["tagName"] as String,
- map["text"] as String,
- map["outerHtml"] as String);
-
@override
- String toString() => "$uri (${span.start.line + 1}:${span.start.column})";
-
- Map toMap() => {
- "uri": uri.toString(),
- "span": _serializeSpan(span),
- "tagName": tagName,
- "text": text,
- "outerHtml": outerHtml
- };
+ String toString() => '$uri (${span.start.line + 1}:${span.start.column})';
}
-
-Map _serializeSpan(SourceSpan span) => {
- "start": _serializeSourceLocation(span.start),
- "end": _serializeSourceLocation(span.end),
- "text": span.text
- };
-
-SourceSpan _deserializeSpan(Map map) => SourceSpan(
- _deserializeSourceLocation(map["start"] as Map),
- _deserializeSourceLocation(map["end"] as Map),
- map["text"] as String);
-
-Map _serializeSourceLocation(SourceLocation location) =>
- {
- "offset": location.offset,
- "line": location.line,
- "column": location.column,
- "sourceUrl": location.sourceUrl.toString()
- };
-
-SourceLocation _deserializeSourceLocation(Map map) =>
- SourceLocation(map["offset"] as int,
- sourceUrl: Uri.parse(map["sourceUrl"] as String),
- line: map["line"] as int,
- column: map["column"] as int);
diff --git a/lib/src/parsers/css.dart b/lib/src/parsers/css.dart
index a76ed12..5dc11cc 100644
--- a/lib/src/parsers/css.dart
+++ b/lib/src/parsers/css.dart
@@ -1,17 +1,12 @@
-library linkcheck.parsers.css;
-
import 'package:csslib/parser.dart' as css;
import 'package:csslib/parser.dart';
import 'package:csslib/visitor.dart';
import 'package:source_span/source_span.dart';
-import '../worker/fetch_results.dart';
import '../destination.dart';
import '../link.dart';
import '../origin.dart';
-import 'package:logging/logging.dart';
-
-Logger _log = Logger('parseCSS');
+import '../worker/fetch_results.dart';
FetchResults parseCss(
String content, Destination current, DestinationResult checked) {
@@ -28,24 +23,18 @@ FetchResults parseCss(
start = content.indexOf("}", start);
if (start < content.length - 1) start += 1;
}
- StyleSheet style;
- try {
- style = css.parse(content.substring(start), errors: errors);
- } catch (e) {
- // csslib itself crashes when trying to parse this.
- // TODO: remove when https://github.com/dart-lang/csslib/issues/92
- // is fixed.
- _log.severe('Parsing ${current.url} crashed csslib');
- break;
- }
+ var style = css.parse(content.substring(start), errors: errors);
style.visit(urlHarvester);
- int offset = 0;
- errors.forEach((error) {
+ var offset = 0;
+ for (var error in errors) {
if (error.level == MessageLevel.severe) {
- offset = error.span.end.offset;
- foundError = true;
+ var errorSpan = error.span;
+ if (errorSpan != null) {
+ offset = errorSpan.end.offset;
+ foundError = true;
+ }
}
- });
+ }
start += offset;
} while (foundError);
@@ -57,7 +46,7 @@ FetchResults parseCss(
// Valid URLs can be surrounded by spaces.
var url = reference.url.trim();
- Link link;
+ Link? link;
// Deal with unsupported schemes such as `telnet:` or `mailto:`.
if (!checkSchemeSupported(url, current.finalUri)) {
@@ -89,7 +78,7 @@ FetchResults parseCss(
continue;
}
- Destination destination = Destination(destinationUri);
+ var destination = Destination(destinationUri);
currentDestinations.add(destination);
link = Link(origin, destination, null);
links.add(link);
@@ -100,16 +89,20 @@ FetchResults parseCss(
}
class CssReference {
- SourceSpan span;
- String url;
+ final SourceSpan span;
+ final String url;
+
CssReference(this.span, this.url);
}
class CssUrlHarvester extends Visitor {
- List references = [];
+ final List references = [];
@override
void visitUriTerm(UriTerm node) {
- references.add(CssReference(node.span, node.text));
+ var span = node.span;
+ if (span != null) {
+ references.add(CssReference(span, node.text));
+ }
}
}
diff --git a/lib/src/parsers/html.dart b/lib/src/parsers/html.dart
index 8357510..2260a19 100644
--- a/lib/src/parsers/html.dart
+++ b/lib/src/parsers/html.dart
@@ -1,5 +1,3 @@
-library linkcheck.parsers.html;
-
import 'package:html/dom.dart';
import 'package:html/parser.dart';
@@ -18,16 +16,20 @@ import '../worker/fetch_results.dart';
/// Setting [parseable] to true will create a link to a destination with
/// [Destination.isSource] set to `true`. For example, links in are
/// often parseable (they are HTML), links in ![]()
often aren't.
-Link extractLink(
- Uri originUri,
- Uri baseUri,
- Element element,
- final List attributes,
- final List destinations,
- bool parseable) {
- var origin = Origin(originUri, element.sourceSpan, element.localName,
- element.text, element.outerHtml);
- String reference;
+Link extractLink(Uri originUri, Uri baseUri, Element element,
+ final List attributes, final List destinations,
+ {bool parseable = false}) {
+ var sourceSpan = element.sourceSpan;
+ var localName = element.localName;
+
+ if (sourceSpan == null || localName == null) {
+ throw StateError(
+ 'Element $element is missing a ${#sourceSpan} or ${#localName}.');
+ }
+
+ var origin =
+ Origin(originUri, sourceSpan, localName, element.text, element.outerHtml);
+ String? reference;
for (var attributeName in attributes) {
reference = element.attributes[attributeName];
if (reference != null) break;
@@ -98,21 +100,26 @@ FetchResults parseHtml(String content, Uri uri, Destination current,
var anchors = doc
.querySelectorAll("body [id], body [name]")
.map((element) => element.attributes["id"] ?? element.attributes["name"])
+ .whereType()
.map(normalizeAnchor)
- .toList();
+ .toList(growable: false);
checked.anchors = anchors;
if (ignoreLinks) {
checked.wasParsed = true;
- return FetchResults(checked, const []);
+ return FetchResults(checked);
}
Uri baseUri = current.finalUri;
var baseElements = doc.querySelectorAll("base[href]");
if (baseElements.isNotEmpty) {
- // More than one base element per page is not according to HTML specs.
- // At the moment, we just ignore that. But TODO: solve for pages with more
- baseUri = baseUri.resolve(baseElements.first.attributes["href"]);
+ var firstBaseHref = baseElements.first.attributes["href"];
+
+ if (firstBaseHref != null) {
+ // More than one base element per page is not according to HTML specs.
+ // At the moment, we just ignore that. But TODO: solve for pages with more
+ baseUri = baseUri.resolve(firstBaseHref);
+ }
}
var linkElements = doc.querySelectorAll(
@@ -123,7 +130,8 @@ FetchResults parseHtml(String content, Uri uri, Destination current,
/// TODO: add destinations to queue, but NOT as a side effect inside extractLink
List links = linkElements
.map((element) => extractLink(current.finalUri, baseUri, element,
- const ["href", "src"], currentDestinations, true))
+ const ["href", "src"], currentDestinations,
+ parseable: true))
.toList();
// Find resources
@@ -131,7 +139,7 @@ FetchResults parseHtml(String content, Uri uri, Destination current,
doc.querySelectorAll("link[href], [src], object[data]");
Iterable currentResourceLinks = resourceElements.map((element) =>
extractLink(current.finalUri, baseUri, element,
- const ["src", "href", "data"], currentDestinations, false));
+ const ["src", "href", "data"], currentDestinations));
links.addAll(currentResourceLinks);
diff --git a/lib/src/parsers/robots_txt.dart b/lib/src/parsers/robots_txt.dart
index 8d13eef..6d06f7b 100644
--- a/lib/src/parsers/robots_txt.dart
+++ b/lib/src/parsers/robots_txt.dart
@@ -1,5 +1,6 @@
-library linkcheck.parsers.robots_txt;
+import 'package:meta/meta.dart';
+@immutable
class RobotsBouncer {
/// The shortest possible identifying part of the user agent.
///
@@ -13,8 +14,6 @@ class RobotsBouncer {
const userAgentString = "User-agent:";
const disallowString = "Disallow:";
- assert(robotName != null);
-
Set currentUserAgents = {};
Set currentPaths = {};
for (var line in lines) {
@@ -64,9 +63,11 @@ class RobotsBouncer {
}
}
+@immutable
class _Rule {
final Set userAgents;
final Set paths;
+
_Rule(this.userAgents, this.paths);
// 'Disallow:' (with empty rulepath) means something like 'allow all'
diff --git a/lib/src/parsers/url_skipper.dart b/lib/src/parsers/url_skipper.dart
index 546b1a3..93d036a 100644
--- a/lib/src/parsers/url_skipper.dart
+++ b/lib/src/parsers/url_skipper.dart
@@ -1,5 +1,3 @@
-library linkcheck.parsers.url_skipper;
-
const _commentStart = "#";
const _commentStartEscape = r"\#";
@@ -7,7 +5,7 @@ const _commentStartEscape = r"\#";
/// Parses and keeps record of the URL patterns to skip.
class UrlSkipper {
/// Path of the provided file with regexps to skip.
- final String path;
+ final String? path;
final List<_UrlSkipperRecord> _records;
@@ -40,7 +38,7 @@ class UrlSkipper {
}
static Iterable<_UrlSkipperRecord> _parse(Iterable lines) sync* {
- int lineNumber = 1;
+ var lineNumber = 1;
for (var line in lines) {
line = line.trim();
diff --git a/lib/src/server_info.dart b/lib/src/server_info.dart
index 92bdea2..241d7e4 100644
--- a/lib/src/server_info.dart
+++ b/lib/src/server_info.dart
@@ -1,4 +1,4 @@
-library linkcheck.server_info;
+import 'package:meta/meta.dart';
import 'parsers/robots_txt.dart';
@@ -11,11 +11,11 @@ class ServerInfo {
/// No duration.
static const immediate = Duration();
- String host;
+ final String host;
- int port;
+ final int? port;
- RobotsBouncer bouncer;
+ RobotsBouncer? bouncer;
/// The total count of connection attempts, both successful and failed.
int connectionAttempts = 0;
@@ -39,7 +39,7 @@ class ServerInfo {
/// or 405).
bool hasFailedHeadRequest = false;
- DateTime _lastRequest;
+ DateTime? _lastRequest;
ServerInfo(String authority)
: host = authority.split(':').first,
@@ -57,9 +57,10 @@ class ServerInfo {
/// Creates the minimum duration to wait before the server should be bothered
/// again.
Duration getThrottlingDuration() {
- if (_lastRequest == null) return immediate;
+ var lastRequest = _lastRequest;
+ if (lastRequest == null) return immediate;
if (isLocalhost) return immediate;
- var sinceLastRequest = DateTime.now().difference(_lastRequest);
+ var sinceLastRequest = DateTime.now().difference(lastRequest);
if (sinceLastRequest.isNegative) {
// There's a request scheduled in the future.
return -sinceLastRequest + minimumDelay;
@@ -83,7 +84,7 @@ class ServerInfo {
forRobot: robotName);
}
- void updateFromStatusCode(int statusCode) {
+ void updateFromStatusCode(int? statusCode) {
connectionAttempts += 1;
if (statusCode == null) {
didNotConnectCount += 1;
@@ -107,21 +108,14 @@ class ServerInfo {
}
/// To be sent from Worker to main thread.
+@immutable
class ServerInfoUpdate {
- String host;
- bool didNotConnect = false;
- String robotsTxtContents = "";
-
- ServerInfoUpdate(this.host);
- ServerInfoUpdate.fromMap(Map map)
- : this._(map["host"] as String, map["robots"] as String,
- map["didNotConnect"] as bool);
-
- ServerInfoUpdate._(this.host, this.robotsTxtContents, this.didNotConnect);
-
- Map toMap() => {
- "host": host,
- "robots": robotsTxtContents,
- "didNotConnect": didNotConnect
- };
+ final String host;
+ final bool didNotConnect;
+ final String robotsTxtContents;
+
+ ServerInfoUpdate(this.host,
+ {this.robotsTxtContents = '', this.didNotConnect = false});
+
+ ServerInfoUpdate.didNotConnect(String host) : this(host, didNotConnect: true);
}
diff --git a/lib/src/uri_glob.dart b/lib/src/uri_glob.dart
index ff11004..64410ad 100644
--- a/lib/src/uri_glob.dart
+++ b/lib/src/uri_glob.dart
@@ -1,8 +1,8 @@
-library linkcheck.uri_glob;
-
import 'package:glob/glob.dart';
+import 'package:meta/meta.dart';
import 'package:path/path.dart';
+@immutable
class UriGlob {
static final _urlContext = Context(style: Style.url);
diff --git a/lib/src/worker/fetch_options.dart b/lib/src/worker/fetch_options.dart
index ada4eb9..bdb7ef6 100644
--- a/lib/src/worker/fetch_options.dart
+++ b/lib/src/worker/fetch_options.dart
@@ -1,5 +1,3 @@
-library linkcheck.fetch_options;
-
import 'dart:async';
import '../uri_glob.dart';
@@ -11,7 +9,7 @@ class FetchOptions {
final headIncompatible = {}; // TODO: send to main
// TODO: hashmap of known problematic servers etc. = List
- final StreamSink> _sink;
+ final StreamSink _sink;
FetchOptions(this._sink);
@@ -22,7 +20,7 @@ class FetchOptions {
}
void info(String message) {
- _sink.add({verbKey: infoFromWorkerVerb, dataKey: message});
+ _sink.add(WorkerTask(verb: WorkerVerb.infoFromWorker, data: message));
}
/// Returns true if the provided [uri] should be considered internal. This
diff --git a/lib/src/worker/fetch_results.dart b/lib/src/worker/fetch_results.dart
index 8e3866f..1f8a825 100644
--- a/lib/src/worker/fetch_results.dart
+++ b/lib/src/worker/fetch_results.dart
@@ -1,22 +1,14 @@
-library linkcheck.fetch_results;
-
import '../destination.dart';
import '../link.dart';
class FetchResults {
final DestinationResult checked;
final List links;
- FetchResults(this.checked, this.links);
- FetchResults.fromMap(Map map)
- : this(
- DestinationResult.fromMap(map["checked"] as Map),
- List.from((map["links"] as List