diff --git a/lib/l10n/app_en.arb b/lib/l10n/app_en.arb index 1a51099b..4d827870 100644 --- a/lib/l10n/app_en.arb +++ b/lib/l10n/app_en.arb @@ -326,6 +326,8 @@ "@database": {}, "databaseSettings": "Database settings", "@databaseSettings": {}, + "scrapeAbstracts": "Scrape missing abstracts", + "@scrapeAbstracts":{}, "cleanupInterval": "Cleanup interval (days)", "@cleanupInterval": {}, "cleanupIntervalHint": "Enter number of days (1 to 365)", diff --git a/lib/screens/article_screen.dart b/lib/screens/article_screen.dart index 65191602..bd007e69 100644 --- a/lib/screens/article_screen.dart +++ b/lib/screens/article_screen.dart @@ -8,7 +8,9 @@ import '../widgets/publication_card.dart'; import './journals_details_screen.dart'; import '../services/zotero_api.dart'; import '../services/string_format_helper.dart'; +import '../services/abstract_scraper.dart'; import 'package:share_plus/share_plus.dart'; +import 'package:shared_preferences/shared_preferences.dart'; class ArticleScreen extends StatefulWidget { final String doi; @@ -22,6 +24,7 @@ class ArticleScreen extends StatefulWidget { final String license; final String licenseName; final String? publisher; + final VoidCallback? onAbstractChanged; const ArticleScreen({ Key? key, @@ -36,6 +39,7 @@ class ArticleScreen extends StatefulWidget { required this.license, required this.licenseName, this.publisher, + this.onAbstractChanged, }) : super(key: key); @override @@ -45,12 +49,17 @@ class ArticleScreen extends StatefulWidget { class _ArticleScreenState extends State { bool isLiked = false; late DatabaseHelper databaseHelper; + String? abstract; + bool isLoadingAbstract = false; + bool _scrapeAbstracts = true; @override void initState() { super.initState(); databaseHelper = DatabaseHelper(); + _loadScrapingSettings(); checkIfLiked(); + abstract = widget.abstract; } void _onShare(BuildContext context) async { @@ -65,6 +74,58 @@ class _ArticleScreenState extends State { } } + Future _loadScrapingSettings() async { + SharedPreferences prefs = await SharedPreferences.getInstance(); + setState(() { + _scrapeAbstracts = prefs.getBool('scrapeAbstracts') ?? true; + }); + if (_scrapeAbstracts) { + final missingAbstractText = + AppLocalizations.of(context)!.abstractunavailable; + + if (abstract == null || + abstract!.isEmpty || + abstract == missingAbstractText) { + fetchAbstract(); + } + } + } + + Future fetchAbstract() async { + if (!mounted) return; + setState(() { + isLoadingAbstract = true; + }); + + //debugPrint("Calling scraper for: ${widget.url}"); + + AbstractScraper scraper = AbstractScraper(); + String? scraped; + try { + scraped = await scraper.scrapeAbstract(widget.url); + } catch (e) { + scraped = ''; + } + String finalAbstract = ''; + if (scraped != null && scraped.isNotEmpty) { + finalAbstract = scraped; + try { + databaseHelper.updateArticleAbstract(widget.doi, finalAbstract); + widget.onAbstractChanged!(); + } catch (e) { + debugPrint("Unable to update the abstract: ${e}"); + } + } else { + finalAbstract = AppLocalizations.of(context)!.abstractunavailable; + } + if (!mounted) return; + setState(() { + abstract = finalAbstract; + isLoadingAbstract = false; + }); + // debugPrint("Final Abstract: $abstract"); + } + @override Widget build(BuildContext context) { return Scaffold( @@ -106,13 +167,27 @@ class _ArticleScreenState extends State { SelectableText(getAuthorsNames(widget.authors), style: TextStyle(color: Colors.grey, fontSize: 15)), SizedBox(height: 15), - SelectableText( + isLoadingAbstract + ? Center(child: CircularProgressIndicator()) + : abstract != null && abstract!.isNotEmpty + ? SelectableText( + abstract!, + textAlign: TextAlign.justify, + style: TextStyle(fontSize: 16), + ) + : Text( + AppLocalizations.of(context)!.abstractunavailable, + textAlign: TextAlign.justify, + style: TextStyle(fontSize: 16), + ), + + /*SelectableText( widget.abstract.isNotEmpty ? widget.abstract : AppLocalizations.of(context)!.abstractunavailable, textAlign: TextAlign.justify, style: TextStyle(fontSize: 16), - ), + ),*/ SizedBox(height: 20), Row( children: [ diff --git a/lib/screens/database_settings_screen.dart b/lib/screens/database_settings_screen.dart index afbd691f..c636ca35 100644 --- a/lib/screens/database_settings_screen.dart +++ b/lib/screens/database_settings_screen.dart @@ -18,6 +18,7 @@ class DatabaseSettingsScreen extends StatefulWidget { class _DatabaseSettingsScreenState extends State { final _formKey = GlobalKey(); + bool _scrapeAbstracts = true; // Default to scraping missing abstracts int _cleanupInterval = 7; // Default for cleanup interval int _fetchInterval = 6; // Default API fetch to 6 hours TextEditingController _cleanupIntervalController = TextEditingController(); @@ -35,6 +36,7 @@ class _DatabaseSettingsScreenState extends State { // Load the values from SharedPreferences if available _cleanupInterval = prefs.getInt('cleanupInterval') ?? 7; _fetchInterval = prefs.getInt('fetchInterval') ?? 6; + _scrapeAbstracts = prefs.getBool('scrapeAbstracts') ?? true; }); _cleanupIntervalController.text = _cleanupInterval.toString(); } @@ -45,6 +47,7 @@ class _DatabaseSettingsScreenState extends State { SharedPreferences prefs = await SharedPreferences.getInstance(); await prefs.setInt('cleanupInterval', _cleanupInterval); await prefs.setInt('fetchInterval', _fetchInterval); + await prefs.setBool('scrapeAbstracts', _scrapeAbstracts); ScaffoldMessenger.of(context).showSnackBar( SnackBar(content: Text(AppLocalizations.of(context)!.settingsSaved)), @@ -221,6 +224,22 @@ class _DatabaseSettingsScreenState extends State { ), ], ), + Row( + mainAxisAlignment: MainAxisAlignment.spaceBetween, + children: [ + Text( + AppLocalizations.of(context)!.scrapeAbstracts, + ), + Switch( + value: _scrapeAbstracts, + onChanged: (bool value) async { + setState(() { + _scrapeAbstracts = value; + }); + }, + ), + ], + ), const SizedBox(height: 16), ElevatedButton( onPressed: _saveSettings, diff --git a/lib/screens/display_settings_screen.dart b/lib/screens/display_settings_screen.dart index 6226d683..be9b81ab 100644 --- a/lib/screens/display_settings_screen.dart +++ b/lib/screens/display_settings_screen.dart @@ -12,7 +12,7 @@ class DisplaySettingsScreen extends StatefulWidget { } class _DisplaySettingsScreenState extends State { - int _publicationCardOption = 0; + int _publicationCardOption = 1; @override void initState() { @@ -24,7 +24,7 @@ class _DisplaySettingsScreenState extends State { final prefs = await SharedPreferences.getInstance(); setState(() { _publicationCardOption = prefs.getInt('publicationCardAbstractSetting') ?? - 0; // Default to "show all abstracts" + 1; // Default to "hide missing abstracts" }); } diff --git a/lib/screens/favorites_screen.dart b/lib/screens/favorites_screen.dart index 4de87a98..611ef874 100644 --- a/lib/screens/favorites_screen.dart +++ b/lib/screens/favorites_screen.dart @@ -15,6 +15,7 @@ class FavoritesScreen extends StatefulWidget { class _FavoritesScreenState extends State { late Future> _favoriteArticles; + late ScrollController _scrollController; int sortBy = 0; // Set the sort by option to Article title by default int sortOrder = 0; // Set the sort order to Ascending by default Map abstractCache = {}; // Cache for abstracts @@ -27,6 +28,7 @@ class _FavoritesScreenState extends State { @override void initState() { super.initState(); + _scrollController = ScrollController(); _favoriteArticles = _loadFavoriteArticles(); _filterController.addListener(() { @@ -36,12 +38,20 @@ class _FavoritesScreenState extends State { Future> _loadFavoriteArticles() async { try { + final double previousOffset = _scrollController.hasClients + ? _scrollController.offset + : 0; // Save scroll position List favorites = await DatabaseHelper().getFavoriteArticles(); setState(() { _allFavorites = favorites; _filteredFavorites = _sortFavorites(favorites); // Apply sorting }); + WidgetsBinding.instance.addPostFrameCallback((_) { + if (_scrollController.hasClients) { + _scrollController.jumpTo(previousOffset); // Restore scroll position + } + }); return favorites; } catch (e) { throw Exception('Failed to load favorite articles: $e'); @@ -156,6 +166,7 @@ class _FavoritesScreenState extends State { ), ) : ListView.builder( + controller: _scrollController, itemCount: _filteredFavorites.length, itemBuilder: (context, index) { final publicationCard = _filteredFavorites[index]; @@ -179,6 +190,9 @@ class _FavoritesScreenState extends State { onFavoriteChanged: () { _removeFavorite(context, publicationCard); }, + onAbstractChanged: () { + _updateAbstract(); + }, ); } else { // Use the AbstractHelper to get the formatted abstract @@ -218,6 +232,9 @@ class _FavoritesScreenState extends State { onFavoriteChanged: () { _removeFavorite(context, publicationCard); }, + onAbstractChanged: () { + _updateAbstract(); + }, ); } }, @@ -328,9 +345,17 @@ class _FavoritesScreenState extends State { ); } + Future _updateAbstract() async { + setState(() { + abstractCache = {}; + _favoriteArticles = _loadFavoriteArticles(); + }); + } + @override void dispose() { _filterController.dispose(); + _scrollController.dispose(); super.dispose(); } } diff --git a/lib/screens/home_screen.dart b/lib/screens/home_screen.dart index d439b5f8..af645926 100644 --- a/lib/screens/home_screen.dart +++ b/lib/screens/home_screen.dart @@ -34,12 +34,18 @@ class _HomeScreenState extends State { final FeedService _feedService = FeedService(); List> savedQueries = []; + bool _feedLoaded = false; // Needed to avoid conflicts wih onAbstractChanged @override void initState() { super.initState(); _loadFetchInterval(); _buildAndStreamFeed(); + WidgetsBinding.instance.addPostFrameCallback((_) { + if (_feedLoaded) { + _onAbstractChanged(); + } + }); _filterController.addListener(() { _filterFeed(_filterController.text); @@ -95,12 +101,13 @@ class _HomeScreenState extends State { if (mounted) { final List cachedFeed = - await _feedService.getCachedFeed(context); + await _feedService.getCachedFeed(context, _onAbstractChanged); setState(() { _allFeed = List.from(cachedFeed); _filteredFeed = List.from(_allFeed); _sortFeed(); + _feedLoaded = true; }); _feedStreamController.add(_filteredFeed); @@ -181,6 +188,17 @@ class _HomeScreenState extends State { }); } + void _onAbstractChanged() async { + final List cachedFeed = + await _feedService.getCachedFeed(context, _onAbstractChanged); + setState(() { + _allFeed = List.from(cachedFeed); + _filterFeed(_filterController.text); + _sortFeed(); + _feedStreamController.add(_filteredFeed); + }); + } + void handleMenuButton(int item) { switch (item) { case 0: diff --git a/lib/services/abstract_helper.dart b/lib/services/abstract_helper.dart index 15b9be79..e44873c1 100644 --- a/lib/services/abstract_helper.dart +++ b/lib/services/abstract_helper.dart @@ -7,7 +7,7 @@ enum AbstractSetting { showAll, hideAll, hideMissing } class AbstractHelper { static Future getAbstractSetting() async { final prefs = await SharedPreferences.getInstance(); - final int setting = prefs.getInt('publicationCardAbstractSetting') ?? 0; + final int setting = prefs.getInt('publicationCardAbstractSetting') ?? 1; switch (setting) { case 1: diff --git a/lib/services/abstract_scraper.dart b/lib/services/abstract_scraper.dart new file mode 100644 index 00000000..e5f3025e --- /dev/null +++ b/lib/services/abstract_scraper.dart @@ -0,0 +1,82 @@ +import 'dart:async'; +import 'package:flutter_inappwebview/flutter_inappwebview.dart'; + +class AbstractScraper { + Completer _completer = Completer(); + + Future scrapeAbstract(String url) async { + _completer = Completer(); + + HeadlessInAppWebView? headlessWebView; + + headlessWebView = HeadlessInAppWebView( + initialSettings: InAppWebViewSettings( + userAgent: + "Mozilla/5.0 (Android 15; Mobile; rv:133.0) Gecko/133.0 Firefox/133.0", + ), + initialUrlRequest: URLRequest(url: WebUri(url)), + onLoadStop: (controller, loadedUrl) async { + await Future.delayed(Duration(seconds: 3)); // Allow JS-loaded content + + if (_completer.isCompleted) return; + + try { + String? abstractText = await controller.evaluateJavascript( + source: """ + (() => { + function extractFullText(element) { + if (!element) return null; + return element.innerText.trim(); + } + + // Special case: Springer (Abstract is inside c-article-section__content) + let springerHeader = document.querySelector('h2.c-article-section__title'); + if (springerHeader && /abstract/i.test(springerHeader.innerText)) { + let springerAbstract = springerHeader.nextElementSibling; + if (springerAbstract && springerAbstract.classList.contains('c-article-section__content')) { + return extractFullText(springerAbstract); + } + } + + // Special case: Elsevier (abstract is inside '.abstract.author') + let elsevierAbstract = document.querySelector('.abstract.author'); + if (elsevierAbstract) { + return extractFullText(elsevierAbstract); + } + + // General case: Look for any div/section with 'abstract' in class or id + let abstractDiv = [...document.querySelectorAll('div, section')] + .find(el => /abstract/i.test(el.className) || /abstract/i.test(el.id)); + + if (abstractDiv) { + return extractFullText(abstractDiv); + } + + // Fallback to meta description + let metaDesc = document.querySelector('meta[name="description"]'); + if (metaDesc) return metaDesc.content.trim(); + + return null; + })(); + """, + ); + abstractText = abstractText! + .replaceAll(RegExp(r'<[^>]*>'), '') // Remove HTML tags + .replaceAll(RegExp(r'^\s*abstract[:.\s]*', caseSensitive: false), + '') // Remove leading "Abstract" + .trim(); + + _completer.complete(abstractText); + } catch (e) { + _completer.completeError(e); + } finally { + await InAppWebViewController.clearAllCache(); + headlessWebView?.dispose(); + } + }, + ); + + await headlessWebView.run(); + return _completer.future; + } +} diff --git a/lib/services/database_helper.dart b/lib/services/database_helper.dart index d9835a98..fbc97b33 100644 --- a/lib/services/database_helper.dart +++ b/lib/services/database_helper.dart @@ -407,16 +407,18 @@ class DatabaseHelper { }); } - /*Future clearCachedPublications() async { + // Updates the abstract of an article after being scraped + Future updateArticleAbstract(String doi, String abstract) async { final db = await database; - await db.delete( + await db.update( 'articles', - where: - 'dateCached IS NOT NULL AND (dateLiked IS NULL AND dateDownloaded IS NULL)', + { + 'abstract': abstract, + }, + where: 'doi = ?', + whereArgs: [doi], ); - }*/ - -// Functions for downloaded articles + } Future isArticleDownloaded(String doi) async { final db = await database; diff --git a/lib/services/feed_service.dart b/lib/services/feed_service.dart index 5d54e4ec..801fc802 100644 --- a/lib/services/feed_service.dart +++ b/lib/services/feed_service.dart @@ -157,7 +157,8 @@ class FeedService { } } - Future> getCachedFeed(BuildContext context) async { + Future> getCachedFeed( + BuildContext context, VoidCallback? onAbstractChanged) async { final cachedPublications = await _dbHelper.getCachedPublications(); return Future.wait(cachedPublications.map((item) async { @@ -172,6 +173,7 @@ class FeedService { url: item.url, license: item.license, licenseName: item.licenseName, + onAbstractChanged: onAbstractChanged, ); }).toList()); } diff --git a/lib/widgets/publication_card.dart b/lib/widgets/publication_card.dart index b37c0171..5a37420e 100644 --- a/lib/widgets/publication_card.dart +++ b/lib/widgets/publication_card.dart @@ -29,6 +29,7 @@ class PublicationCard extends StatefulWidget { final String licenseName; final String? dateLiked; final VoidCallback? onFavoriteChanged; + final VoidCallback? onAbstractChanged; final String? publisher; const PublicationCard({ @@ -45,6 +46,7 @@ class PublicationCard extends StatefulWidget { required this.licenseName, this.dateLiked, this.onFavoriteChanged, + this.onAbstractChanged, this.publisher, }) : authors = authors, super(key: key); @@ -90,6 +92,9 @@ class _PublicationCardState extends State { license: widget.license, licenseName: widget.licenseName, publisher: widget.publisher, + onAbstractChanged: () { + widget.onAbstractChanged!(); + }, ), ), );