import instascrape
scraper = instascrape.InstagramScraper()
usernames = ["lonepsi", "ragdoll_brotherz", "kendricklamar"]
for username in usernames:
media = scraper.scrape_user_medias(username, count=10)
for photo in media:
print(f"Média ID: {photo['id']}")
print(f"Propriétaire du média: {photo['owner']['username']}")
print(f"Description: {photo['caption']}")
print(f"Nombre de likes: {photo['like_count']}")
print(f"Date de création: {photo['taken_at_timestamp']}")
I'm trying to collect data on instagram with instascrape but when I run my code here is what I get as a response :
from helpers import extract_email
ImportError: cannot import name 'extract_email' from 'helpers' (D:\Users\Paul\Documents\inge_3\projet_insta\post-acquisition\venv\lib\site-packages\helpers\__init__.py)
i tried several solutions but nothing worked
Related
I write code for Replit.com
i make a command /add_administrator that is, Ana must add the user ID of the user I mentioned to the file admins.json but after using the command the bot breaks What I understand the bot sees(/add_administrator)
(#user)ignore or not see
I would like to understand how to get the user id through a #mention
It will be in a different language full code :https://github.com/jonoto2/TeleBotPHP/blob/main/main.py
complete command code:
from webserver import keep_alive
import json
import random
import re
import os
import telebot
from telebot import types
#from telegram.ext import Updater
bot = telebot.TeleBot(os.environ['TOKEN'], parse_mode="HTML")
#bot.message_handler(commands=['add_adm'])
def add_adm(message):
# Check if user is admin
if is_admin(int(message.from_user.id)):
args = get_args(message.text)
# Check for args
if args and args[0].isdigit():
id = int(args[0])
with open('admins.json', "r") as jsonFile:
data = json.load(jsonFile)
if id not in data:
data.append(id)
with open("admins.json", "w") as jsonFile:
json.dump(data, jsonFile)
bot.reply_to(message, f"`{id}` Appointed as admin.")
else:
bot.reply_to(message,
'This user is already an administrator .')
else:
bot.reply_to(message, 'User is not found.')
else:
bot.reply_to(message, 'You are not an administrator .')
I have not tried anything because I do not understand how
I am struggeling to return a file e.g. a pdf which was uploaded to a mongodb.
I am able to upload a file to the database but I am not able to retrieve the file again.
How shoud my endpoint (view) look like to return the file?
I am using django rest framework v3.12.4 with djongo v1.3.6. I use drf-yasg v1.20.0 for the documentation of the API.
Here are my settings, models.py, serializers.py, views.py and urls.py:
# app settings.py
DATABASES = {
'default': {
'ENGINE': 'djongo',
'NAME': 'TAST_DB2',
'CLIENT': {
'host': 'localhost',
'port': 27017,
'username': 'root',
'password': 'reallychangeme', # :-)
'authSource': 'admin',
'authMechanism': 'SCRAM-SHA-1'
}
}
}
DEFAULT_FILE_STORAGE = 'mongo_storage.storage.GridFSStorage'
GRIDFS_DATABASE = 'myfiles'
BASE_URL = 'http://localhost:8085/'
UPLOADED_FILES_USE_URL = True
# models.py
from django.db import models
from djongo.storage import GridFSStorage
grid_fs_storage = GridFSStorage(collection='myfiles', base_url=''.join([settings.BASE_URL, 'myfiles/']))
class TestStandardFile(models.Model):
myfile = models.FileField(upload_to='teststandards1', storage=grid_fs_storage)
# serializers.py
from rest_framework import serializers
from teststandards.models import TestStandardFile
class TestStandardFileSerializer(serializers.ModelSerializer):
class Meta:
model = TestStandardFile
fields = '__all__'
# views.py
from rest_framework.generics import ListCreateAPIView
from .models import TestStandard
from .serializers import TestStandardFileSerializer
from rest_framework.parsers import MultiPartParser
# used for the upload
class FileView(ListCreateAPIView):
parser_classes = ( MultiPartParser,)
serializer_class = TestStandardFileSerializer
queryset = TestStandardFile.objects.all()
<---------- !ENDPOINT FOR FILE RETRIEVAL MISSING HERE???
# urls.py
urlpatterns = [
re_path(r'^api/teststandards/file', api.FileView.as_view(), name='teststandard-file'),
re_path(r'^myfiles/(?P<pk>[0-9]+)$', api.myfiles.as_view(), name='get-file'),
]
I can see my file properly uploaded to mongodb with mongodb compass.
There are three collections in it:
TAST_DB2.teststandardsFiles
TAST_DB2.myfiles.teststandards1.files
TAST_DB2.myfiles.teststandards1.chunks
I assume that I need an endpoint which gives the file back as a response.
I tried to overwrite the 'get'-function of my 'myfiles' endpoint. But I don't know
how to get the file handle from the requested file. And I do not know how to
return the file as a HttpResponse.
Any help is appreciated!
I finally got it to work. I created an RetrieveAPIView for the retrieval of one entry and overwrote the retrieve function. THis is how my views.py looked like:
# for upload
class FileView(ListCreateAPIView):
parser_classes = ( MultiPartParser,)
serializer_class = TestStandardFileSerializer
queryset = TestStandardFile.objects.all()
# download
class myfiles(RetrieveAPIView):
parser_classes = ( MultiPartParser,)
serializer_class = TestStandardFileSerializer
queryset = TestStandardFile.objects.all()
def retrieve(self, request, *args, **kwargs):
obj = self.get_object()
response = HttpResponse(obj.myfile, content_type='application/octet-stream')
response['Content-Disposition'] = 'attachment; filename=%s' % obj.myfile
return response
"myfile" is the name of the FileField from my model. With using drf_spectacular for the swagger documentation. This generated a nice download button for the file retrieval. It helped a lot during testing the upload / download functionality.
soap.find(id='productTitle').get_text(strip=True)
Output: 'NoneType' Object has no attribute 'get_text'.
There's not a lot to go off since you didn't provide a lot of information, but from the information I got, you've put soap.find instead of soup.find
You could try something like this to fix it:
import requests
from bs4 import BeautifulSoup
URL = "Your url"
headers = {
"User-Agent": '(search My user agent)'}
def product_title():
req = requests.Session()
page = req.get(URL, headers=headers)
soup = BeautifulSoup(page.content, 'html.parser')
productTitle = soup.find(id='productTitle').get_text(strip=True)
print(product)
productTitle()
i created a new project by django restframework
the project name = frame
th project_app name= framework
fram.urls.py:
"""frame URL Configuration
The `urlpatterns` list routes URLs to views. For more information please see:
https://docs.djangoproject.com/en/3.1/topics/http/urls/
Examples:
Function views
1. Add an import: from my_app import views
2. Add a URL to urlpatterns: path('', views.home, name='home')
Class-based views
1. Add an import: from other_app.views import Home
2. Add a URL to urlpatterns: path('', Home.as_view(), name='home')
Including another URLconf
1. Import the include() function: from django.urls import include, path
2. Add a URL to urlpatterns: path('blog/', include('blog.urls'))
"""
from django.contrib import admin
from django.urls import path,re_path
from django.conf.urls import include,url
from framework.views import myviewset
from rest_framework import routers
router = routers.DefaultRouter();
router.register('task', myviewset)
urlpatterns = [
path('admin/', admin.site.urls),
re_path(r'', include(router.urls)),
]
framework models.py:
from django.db import models
# Create your models here.
class Task (models.Model):
task_name=models.CharField(max_length='200')
task_desc=models.CharField(max_length='200')
date_created=models.DateTimeField(auto_now=True)
serializer.py
from .models import Task
from rest_framework import serializers
class TaskSerializers(serializers.ModelSerializer):
class Meta:
model = Task
fields = ('id', 'task_name', 'task_desc')
views.py:
from django.shortcuts import render
from .serializer import TaskSerializers
from rest_framework import viewsets
from .models import Task
# Create your views here.
class myviewset(viewsets.ModelViewSet):
query = Task.objects.all().order_by('date_created')
serializer_class = TaskSerializers
admin.py:
from django.contrib import admin
from .models import Task
admin.site.register(Task)
# Register your models here.
setting.py:
INSTALLED_APPS = [
'framework',
'rest_framework',
'django.contrib.admin',
'django.contrib.auth',
'django.contrib.contenttypes',
'django.contrib.sessions',
'django.contrib.messages',
'django.contrib.staticfiles',
]
when i run th server in CMD
this error comes :
AssertionError: basename argument not specified, and could not automatically determine the name from the viewset, as it does not have a .queryset attribute.
You should specify queryset attribute instead of query in your viewset. For example (class name case changed to comply PEP8):
class MyViewSet(viewsets.ModelViewSet):
queryset = Task.objects.all().order_by('date_created')
serializer_class = TaskSerializers
Docs: https://www.django-rest-framework.org/api-guide/viewsets/#modelviewset
Also I recommend always specify basename when you register your viewsets. Like this:
router = routers.DefaultRouter();
router.register('task', MyViewSet, basename='tasks')
https://www.django-rest-framework.org/api-guide/routers/#usage
basename - The base to use for the URL names that are created. If unset the basename will be automatically generated based on the queryset attribute of the viewset, if it has one. Note that if the viewset does not include a queryset attribute then you must set basename when registering the viewset.
I'm trying to scrape the live billionaire networth table here > https://www.bloomberg.com/billionaires/
This is my code so far. All I get is [] as result on the python shell.
Something has to be wrong with the "findAll", I don't think I'm using the correct tag lines.
Tried to use just "find"
from urllib.request import urlopen as uReq
from bs4 import BeautifulSoup as soup
import csv
#Open page and grab html
my_url = ('https://www.bloomberg.com/billionaires/')
uClient = uReq(my_url)
page_html = uClient.read()
uClient.close()
#HTML Parser.
page_soup = soup(page_html, 'html.parser')
table = []
#Find table.
ele_table = page_soup.findAll('div',{'class':'dvz-content'})
print(ele_table)
I'm expecting for the table to be printed out so I can get it into a CSV file.
Data is dynamically loaded. You can pull from script tag provided you supply the right headers. Regex out the required info and parse with json library. Hand this off to pandas to write to csv
from bs4 import BeautifulSoup as bs
import requests, re, json
import pandas as pd
headers = {
'user-agent': 'Mozilla/5.0',
'accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3',
'if-none-match': 'W/^\\^5dbb59e6-91b10^\\^',
'if-modified-since': 'Thu, 31 Oct 2019 22:02:14 GMT' # this may be safeguard for caching. Consider if add dynamically.
}
p = re.compile(r'window.top500 = (.*);')
r = requests.get('https://www.bloomberg.com/billionaires/', headers = headers)
data = json.loads(p.findall(r.text)[0])
df = pd.DataFrame(data)
df.to_csv(r'C:\Users\User\Desktop\Data.csv', sep=',', encoding='utf-8-sig',index = False)
Example output: