I want to scrap a lazy loaded website using flutter, i used webscraper package as it only scrapes visible elements, how can I scrap all the images and links from the website.
Please refer the image
class Top2 extends StatefulWidget {
const Top2({Key? key}) : super(key: key);
#override
State<Top2> createState() => _Top2State();
}
class _Top2State extends State<Top2> {
late List<Map<String, dynamic>> top2Wall;
bool top2Loaded = false;
void top2Fetch() async {
final top2Scraper = WebScraper('https://mobile.alphacoders.com');
if (await top2Scraper.loadWebPage('/by-category/3?page=1')) {
top2Wall = top2Scraper.getElement(
'div.container-masonry > div.item > a > img.img-responsive',
['src', 'title']);
// ignore: avoid_print
print(top2Wall);
setState(() {
top2Loaded = true;
});
}
}
#override
void initState() {
super.initState();
top2Fetch();
}
#override
Widget build(BuildContext context) {
Size screenSize = MediaQuery.of(context).size;
return Scaffold(
body: top2Loaded
// ignore: sized_box_for_whitespace
? Container(
height: screenSize.height,
width: double.infinity,
child: SingleChildScrollView(
physics: const BouncingScrollPhysics(),
child: Wrap(children: [
for (int i = 1; i < top2Wall.length; i++)
WallCard(src: top2Wall[i]['attributes']['src'])
]),
),
)
: const Center(
child: CircularProgressIndicator(color: Colors.cyanAccent),
));
}
}
So basically i want to scrap all the wallpaper available from the website but can only scrap the first 24 wallpapers visible.
Its simple.
Rest of the wallpapers are fetched dynamic using JS.
Check Pc Browser > Dev Options> Network and scroll down the page you will se something like:
You can maybe check url where XHR connect and try to scrap it
EDIT
I See page have pagination switch so maybe will be easier to use this
Related
I'm trying to show the full image that has been clicked on from my drawer.
I have a liste of images that I display in my drawer and What I want is that when I click on a specific image, it closes my drawer and show the image on my screen in a kind of an image slider where I can switch images directly from the opened image.
here is my code where I extract my list of images from my asset folder :
import 'dart:convert';
import 'package:flutter/material.dart';
import 'package:flutter/services.dart';
class GetImages extends StatefulWidget {
const GetImages({super.key});
#override
State<GetImages> createState() => _GetImagesState();
}
class _GetImagesState extends State<GetImages> {
List<String> imagePaths = [];
#override
void initState() {
_initImages();
super.initState();
}
Future _initImages() async {
final Map<String, dynamic> assets =
jsonDecode(await rootBundle.loadString('AssetManifest.json'));
setState(() {
imagePaths = assets.keys
.where((String key) => key.contains('photos/'))
.where((String key) => key.contains('.JPG'))
.toList();
});
}
#override
Widget build(BuildContext context) {
return LayoutBuilder(builder: (context, constraints) {
return GridView.count(
crossAxisCount: constraints.maxWidth > 700 ? 4 : 2,
children: imagePaths
.map(
(path) => Padding(
padding: const EdgeInsets.symmetric(horizontal: 17),
child: Image.asset(path),
),
)
.toList(),
);
});
}
}
And here is my code for my drawer :
import 'package:flutter/material.dart';
import 'package:myapp/widgets/get_images.dart';
import 'package:image_viewer/image_viewer.dart';
class SideBar extends StatelessWidget {
const SideBar({super.key, required this.title});
final String title;
#override
Widget build(BuildContext context) {
return Scaffold(
appBar: AppBar(title: Text(title)),
body: const Center(
child: Text('My Page!'),
),
drawer: Drawer(
child: InkWell(
child: GetImages(),
onTap: () {
//ImageViewer.showImageSlider(images: ["assets/photos/IMG_4100.JPG"]);
// montre la photo et ferme la sidebar
Navigator.pop(context);
},
),
),
);
}
}
Thanks in advance for your help :)
You could try this package that i've used before https://pub.dev/packages/lightbox seems like it does exactly what you are looking for.
I'm trying to scrape a website using the package web_scraper, where I want the user to click a button and the new link opens where new scrapped images can be shown.
class Top2 extends StatefulWidget {
const Top2({Key? key}) : super(key: key);
#override
State<Top2> createState() => _Top2State();
}
class _Top2State extends State<Top2> {
late List<Map<String, dynamic>> top2Wall;
bool top2Loaded = false;
int page = 2;
void top2Fetch() async {
final top2Scraper = WebScraper('https://mobile.alphacoders.com');
if (await top2Scraper
.loadWebPage('/by-category/3?page=$page&quickload=1')) {
top2Wall = top2Scraper.getElement('div.item > a > img', ['src', 'title']);
// ignore: avoid_print
print(top2Wall);
setState(() {
top2Loaded = true;
});
}
}
#override
void initState() {
super.initState();
top2Fetch();
}
#override
Widget build(BuildContext context) {
Size screenSize = MediaQuery.of(context).size;
return Scaffold(
body: top2Loaded
// ignore: sized_box_for_whitespace
? Container(
height: screenSize.height,
width: double.infinity,
child: SingleChildScrollView(
physics: const BouncingScrollPhysics(),
child: Wrap(children: [
for (int i = 1; i < top2Wall.length; i++)
WallCard(src: top2Wall[i]['attributes']['src']),
InkWell(
onTap: () {
setState(() {
page++;
// ignore: avoid_print
print(page);
});
},
child: Container(
height: 100,
width: 100,
decoration: const BoxDecoration(color: Colors.cyan)),
)
]),
),
)
: const Center(
child: CircularProgressIndicator(color: Colors.cyanAccent),
));
}[image2][1]
}
For a clear explanation I want to change the page link for scraping, so basically the page number=1 i want to increase it by a number, when user clicks on the container. I used SetState and page++ to increment it by a digit everytime the user clicks on it. I used the print statement to check wether the page increments or not and its sucessfully increases but the page remains same, for clear code view please refer these images enter image description here
enter image description here
If I understand what your issue is, after calling increment page ++ in setstate, you have to call top2Fetch() again inside the onTap method. Like this:
onTap: ()async {
setState(()=>page++);
await top2Fetch();
}
Then update your UI.
You can include a bool value called isLoading in the onTap method: such that when you call the top2Fetch method , a loading spinner is shown on the UI till the future is done.
I wanted to ask this certain question, I don't know if I'm being silly or not.
[https://mobile.alphacoders.com/by-category/10?page=1&quickload=1]
here you can see in the link "page=1", I want to create a button in my flutter app so that when I click on the button the page number increases with one digit
so like page=1+1
I think I can use the '$' property in the URL and add a setState but the trickier thing is I'm trying to scrape a website's wallpaper so when the user clicks on the next page button he must be redirected to a new page with new wallpapers
class Top2 extends StatefulWidget {
const Top2({Key? key}) : super(key: key);
#override
State<Top2> createState() => _Top2State();
}
class _Top2State extends State<Top2> {
late List<Map<String, dynamic>> top2Wall;
bool top2Loaded = false;
void top2Fetch() async {
final top2Scraper = WebScraper('https://mobile.alphacoders.com');
if (await top2Scraper.loadWebPage('/by-category/3?page=1')) {
top2Wall = top2Scraper.getElement(
'div.container-masonry > div.item > a > img.img-responsive',
['src', 'title']);
// ignore: avoid_print
print(top2Wall);
setState(() {
top2Loaded = true;
});
}
}
#override
void initState() {
super.initState();
top2Fetch();
}
#override
Widget build(BuildContext context) {
Size screenSize = MediaQuery.of(context).size;
return Scaffold(
body: top2Loaded
// ignore: sized_box_for_whitespace
? Container(
height: screenSize.height,
width: double.infinity,
child: SingleChildScrollView(
physics: const BouncingScrollPhysics(),
child: Wrap(children: [
for (int i = 1; i < top2Wall.length; i++)
WallCard(src: top2Wall[i]['attributes']['src'])
]),
),
)
: const Center(
child: CircularProgressIndicator(color: Colors.cyanAccent),
));
}
}
please refer to this image for clear code sample
Hello #Cheems Monarch
You can try like this :
First create a variable:
int page=1;
Then change your method like this:
void top2Fetch(String page) async {
final top2Scraper = WebScraper('https://mobile.alphacoders.com');
if (await top2Scraper.loadWebPage('/by-category/3?page=$page')) {
top2Wall = top2Scraper.getElement(
'div.container-masonry > div.item > a > img.img-responsive',
['src', 'title']);
// ignore: avoid_print
print(top2Wall);
setState(() {
top2Loaded = true;
});
}
}
And call this function on your button's onPressed:
onPressed: () {
top2Fetch(page.toString());
setState(() {
page++;
});
}
I am building a method that the user can select a prefered profile picture to show arround the app, using provider package. I used shared_preferences to save the profile picture preferences on locally as a int value. And it worked, means I can save the profile picture to local system. But the problem is, the provider package completely became useless in this case, because I have to convert the widget to statefull and call the setState method when ever I insert a profilePicture widget inside the widget tree. And even the profilePicture widget in the HomeScreen not updating this way. I want to know how can I use the provider package for this issue instead of using statefulWidgets.
watch the Gif or video
This is the Provider class I created:
class ProfilePicProvider with ChangeNotifier {
ProfilePicPref profilePicPreferences = ProfilePicPref();
int _svgNumber = 1;
int get svgNumber => _svgNumber;
set svgNumber(int value) {
_svgNumber = value;
profilePicPreferences.setProfilePic(value);
notifyListeners();
}
void changePic(int val) {
_svgNumber = val;
profilePicPreferences.setProfilePic(val);
notifyListeners();
}
}
This is the sharedPreferences class
class ProfilePicPref {
static const PRO_PIC_STS = 'PROFILESTATUS';
setProfilePic(int svgNo) async {
SharedPreferences profilePref = await SharedPreferences.getInstance();
profilePref.setInt(PRO_PIC_STS, svgNo);
}
Future<int> getProfilePicture() async {
SharedPreferences profilePref = await SharedPreferences.getInstance();
return profilePref.getInt(PRO_PIC_STS) ?? 1;
}
}
This is the image selection screen and save that data to sharedPreferences class
class SelectProfilePicture extends StatefulWidget {
const SelectProfilePicture({Key? key}) : super(key: key);
#override
State<SelectProfilePicture> createState() => _SelectProfilePictureState();
}
class _SelectProfilePictureState extends State<SelectProfilePicture> {
int svgNumber = 1;
ProfilePicProvider proProvider = ProfilePicProvider();
#override
void initState() {
getCurrentProfilePicture();
super.initState();
}
void getCurrentProfilePicture() async {
proProvider.svgNumber =
await proProvider.profilePicPreferences.getProfilePicture();
setState(() {});
}
#override
Widget build(BuildContext context) {
return Scaffold(
body: Column(
children: [
CurrentAccountPicture(
path: 'assets/svg/${proProvider.svgNumber}.svg'),
Expanded(
child: GridView.builder(
itemCount: 15,
gridDelegate: const SliverGridDelegateWithFixedCrossAxisCount(
crossAxisCount: 3,
),
itemBuilder: (context, index) {
return GestureDetector(
onTap: () {
setState(() {
svgNumber = index + 1;
});
proProvider.changePic(index + 1);
proProvider.svgNumber = index + 1;
},
child: SvgPicture.asset('assets/svg/${index + 1}.svg'),
);
},
),
),
],
),
);
}
}
This is the HomeScreen which is not updating the profile image whether it is statefull or stateless
class HomePage extends StatelessWidget {
const HomePage({Key? key}) : super(key: key);
#override
Widget build(BuildContext context) {
final proPicProvider = Provider.of<ProfilePicProvider>(context);
return Scaffold(
body:
Column(
children: [
Row(
children: [
CurrentAccountPicture(
path: 'assets/svg/${proPicProvider.svgNumber}.svg'),
],
),
],
),
);
}
}
example:
I have to convert the widget to statefull and call setState method to get the current profile picture from sharedPreferences. You may find this screen from the GIF I provided.
class Progress extends StatefulWidget {
const Progress({Key? key}) : super(key: key);
#override
State<Progress> createState() => _ProgressState();
}
class _ProgressState extends State<Progress> {
ProfilePicProvider proProvider = ProfilePicProvider();
#override
void initState() {
getCurrentProfilePicture();
super.initState();
}
void getCurrentProfilePicture() async {
proProvider.svgNumber =
await proProvider.profilePicPreferences.getProfilePicture();
setState(() {});
}
#override
Widget build(BuildContext context) {
return Scaffold(
body: SafeArea(
child: SizedBox(
height: 130.0,
width: 130.0,
child: SvgPicture.asset(
'assets/svg/${proProvider.svgNumber}.svg'),
),
),
);
}
}
The problem is in _SelectProfilePictureState when you create new instance of your ChangeNotifier:
ProfilePicProvider proProvider = ProfilePicProvider();. It means you are not using the provider available across the context but creating new one every time. So when the value of your provider changed it has effect only inside _SelectProfilePictureState. Instead of creating new instance you must call it always using the context:
class SelectProfilePicture extends StatefulWidget {
const SelectProfilePicture({Key? key}) : super(key: key);
#override
State<SelectProfilePicture> createState() => _SelectProfilePictureState();
}
class _SelectProfilePictureState extends State<SelectProfilePicture> {
int svgNumber = 1;
// [removed] ProfilePicProvider proProvider = ProfilePicProvider();
//removed
/*void getCurrentProfilePicture() async {
proProvider.svgNumber =
await proProvider.profilePicPreferences.getProfilePicture();
setState(() {});
}*/
#override
Widget build(BuildContext context) {
//use provider from the context
final proProvider = Provider.of<ProfilePicProvider>(context,listen:true);
return Scaffold(
body: Column(
children: [
CurrentAccountPicture(
path: 'assets/svg/${proProvider.svgNumber}.svg'),
Expanded(
child: GridView.builder(
itemCount: 15,
gridDelegate: const SliverGridDelegateWithFixedCrossAxisCount(
crossAxisCount: 3,
),
itemBuilder: (context, index) {
return GestureDetector(
onTap: () {
setState(() {
svgNumber = index + 1;
});
proProvider.changePic(index + 1);
proProvider.svgNumber = index + 1;
},
child: SvgPicture.asset('assets/svg/${index + 1}.svg'),
);
},
),
),
],
),
);
}
}
If you enter the application you may want send initially selected image to your provider:
Add parameter to the constructor of ProfilePicProvider:
ProfilePicProvider(SharedPreferences prefs): _svgNumber = prefs.getInt(ProfilePicPref.PRO_PIC_STS) ?? 1;
In main.dart:
Future<void> main()async{
WidgetsFlutterBinding.ensureInitialized();
var prefs = await SharedPreferences.getInstance();
runApp(
MultiProvider(
providers:[
ChangeNotifierProvider( create:(_) => ProfilePicProvider(prefs))
],
child: yourtopWidget
)
);
}
My Project was working totally fine a month ago, and now after several flutter updates, my iFrame widget is false centered in its desired renderbox. First picture is the wrong behavior, second is the old working version. I'm using a plugin which prevents the analyzer from prompting errors when using platformViewRegistry. Below is my code for the iframe-widget.
Does someone know how to fix this? I don't want to downgrade to older flutter versions.
Thanks for any help!
PS: Simple Center() did not work
My IFrame Widget
// ignore: avoid_web_libraries_in_flutter
import 'dart:html';
// ignore: undefined_prefixed_name
import 'package:universal_ui/universal_ui.dart';
import 'package:flutter/material.dart';
class Iframe extends StatefulWidget {
final String source;
final Size size;
Iframe(this.source, {this.size});
#override
_IframeState createState() => _IframeState();
}
class _IframeState extends State<Iframe> {
Widget _iframeWidget;
String source;
#override
void initState() {
newFrame();
super.initState();
}
void newFrame() async {
print(widget.size);
final String id = widget.source.hashCode.toString();
final IFrameElement _iframeElement = IFrameElement();
_iframeElement.height = widget.size?.height?.toString() ?? '500';
_iframeElement.width = widget.size?.width?.toString() ?? '500';
source = widget.source;
_iframeElement.src = widget.source;
_iframeElement.style.border = 'none';
ui.platformViewRegistry
.registerViewFactory(id, (int viewID) => _iframeElement);
_iframeWidget = HtmlElementView(
key: UniqueKey(),
viewType: id,
);
}
#override
Widget build(BuildContext context) {
if (source != widget.source) newFrame();
return _iframeWidget;
}
}
Using IFrame Widget
class ChatClientAnon extends StatelessWidget {
#override
Widget build(BuildContext context) {
return StreamCacheBuilder<Package>(
stream: Database().streamPackage(),
builder: (data) => Container(
child: Row(
children: [
Expanded(child: SmartphoneClient('Anonym', isAnon: true), flex: 2),
Expanded(
child: LayoutBuilder(
builder: (_, c) =>
Iframe(data.source, size: c.biggest)),
flex: 8),
],
),
),
);
}
}
Update: issue got fixed with next flutter update