Here's what I tried (from one suggestion in this group):
from pandas_datareader import data as dreader
symbols = ['GOOG', 'AAPL', 'MMM', 'ACN', 'A', 'ADP']
pnls = {i:dreader.DataReader(i,'yahoo','1985-01-01','2016-09-01') for i in symbols}
this is the error
RemoteDataError: Unable to read URL: https://finance.yahoo.com/quote/GOOG/history?period1=473380200&period2=1472768999&interval=1d&frequency=1d&filter=history
Here's another code that does not seem to work:
import time
import datetime
import pandas as pd
tickers = ['TSLA', 'TWTR', 'MSFT', 'GOOG', 'AAPL']
interval = '1d'
period1 = int(time.mktime(datetime.datetime(2021, 1, 1, 23, 59).timetuple()))
period2 = int(time.mktime(datetime.datetime(2022, 5, 31, 23, 59).timetuple()))
xlwriter = pd.ExcelWriter('historicalprices.xlsx', engine='openpyxl')
for ticker in tickers:
query_string = 'https://query1.finance.yahoo.com/v7/finance/download/{ticker}?period1={period1}&period2={period2}&interval={interval}&events=history&includeAdjustedClose=true'
df = pd.read_csv(query_string)
df.to_excel(xlwriter, sheet_name=ticker, index = False)
xlwriter.save()
this is the error:
HTTPError: HTTP Error 500: Internal Server Error
This code works for individual stocks:
yahoo_financials = YahooFinancials('AAPL')
data = yahoo_financials.get_historical_price_data(start_date='2021-12-01', end_date='2022-05-31',
time_interval='daily')
aapl_df = pd.DataFrame(data['AAPL']['prices'])
aapl_df = aapl_df.drop('date', axis=1).set_index('formatted_date')
aapl_df.to_csv('/Users/rangapriyas/Desktop/Prices/AAPL.csv')
I am new to python can anyone help me with a for loop on this above code pls?
Thanks in advance!
This code works for anyone who may benefit:
import yfinance as yf
import pandas as pd
tickers_list = ['AAPL', 'WMT', 'IBM', 'MU', 'BA', 'AXP']
# Fetch the data
import yfinance as yf
data = yf.download(tickers_list,'2015-1-1', '2022-05-31')['Adj Close']
# Print first 5 rows of the data
print(data.head())
Related
I am trying to run a code repository downloaded from GitHub as it is mentioned in the instruction of that but getting following error.
TypeError: init() missing 1 required positional argument: 'image_paths'
I am having this error at the code line 63 (preprocessing = preprocessing).
When I srat the program in debug mode I shows following error
unable to get repr for <class 'albumentations.core.composition.compose'>
import matplotlib.pyplot as plt
import numpy as np
import os
import sys
import torch
from skimage import io
from utils import adjust_sar_contrast, compute_building_score, plot_images
sys.path.append('/home/salman/Downloads/SpaceNet_SAR_Buildings_Solutions-master/4-motokimura/tmp/work')
from spacenet6_model.configs.load_config import get_config_with_previous_experiment
from spacenet6_model.datasets import SpaceNet6TestDataset
from spacenet6_model.models import get_model
from spacenet6_model.transforms import get_augmentation, get_preprocess
# select previous experiment to load
exp_id = 14
exp_log_dir = "/home/salman/Downloads/SpaceNet_SAR_Buildings_Solutions-master/4-motokimura/tmp/logs" # None: use default
# select device to which the model is loaded
cuda = True
if cuda:
device = 'cuda'
os.environ['CUDA_VISIBLE_DEVICES'] = '0'
else:
device = 'cpu'
os.environ['CUDA_VISIBLE_DEVICES'] = ''
# overwrite default config with previous experiment
config = get_config_with_previous_experiment(exp_id=exp_id, exp_log_dir=exp_log_dir)
# overwrite additional hyper parameters
config.MODEL.DEVICE = device
config.WEIGHT_ROOT = "/home/salman/Downloads/SpaceNet_SAR_Buildings_Solutions-master/4-motokimura/tmp/weights/"
config.MODEL.WEIGHT = f"/home/salman/Downloads/SpaceNet_SAR_Buildings_Solutions-master/4-motokimura/tmp/weights/exp_{exp_id:04d}/model_best.pth"
config.INPUT.MEAN_STD_DIR = "/home/salman/Downloads/SpaceNet_SAR_Buildings_Solutions-master/4-motokimura/tmp/work/models/image_mean_std/"
config.INPUT.TEST_IMAGE_DIR = "/home/salman/data/SN6_buildings_AOI_11_Rotterdam_test_public/test_public/AOI_11_Rotterdam/SAR-Intensity"
config.INPUT.SAR_ORIENTATION="/home/salman/Downloads/SpaceNet_SAR_Buildings_Solutions-master/4-motokimura/tmp/work/static/SAR_orientations.txt"
config.TRAIN_VAL_SPLIT_DIR="/home/salman/Downloads/data/spacenet6/split"
config.PREDICTION_ROOT="/home/salman/Downloads/data/spacenet6/predictions"
config.POLY_CSV_ROOT="/home/salman/Downloads/data/spacenet6/polygons"
config.CHECKPOINT_ROOT="/home/salman/Downloads/data/spacenet6/ceckpoints"
config.POLY_OUTPUT_PATH="/home/salman/Downloads/data/spacenet6/val_polygons"
config.freeze()
print(config)
model = get_model(config)
model.eval();
from glob import glob
image_paths = glob(os.path.join(config.INPUT.TEST_IMAGE_DIR, "*.tif"))
#print(image_paths)
preprocessing = get_preprocess(config, is_test=True)
augmentation = get_augmentation(config, is_train=False)
test_dataset = SpaceNet6TestDataset(
config,
augmentation=augmentation,
preprocessing=preprocessing
)
test_dataset_vis = SpaceNet6TestDataset(
config,
augmentation=augmentation,
preprocessing=None
)
channel_footprint = config.INPUT.CLASSES.index('building_footprint')
channel_boundary = config.INPUT.CLASSES.index('building_boundary')
score_thresh = 0.5
alpha = 1.0
start_index = 900
N = 20
for i in range(start_index, start_index + N):
image_vis = test_dataset_vis[i]['image']
image = test_dataset[i]['image']
x_tensor = image.unsqueeze(0).to(config.MODEL.DEVICE)
pr_score = model.module.predict(x_tensor)
pr_score = pr_score.squeeze().cpu().numpy()
pr_score_building = compute_building_score(
pr_score[channel_footprint],
pr_score[channel_boundary],
alpha=alpha
)
pr_mask = pr_score_building > score_thresh
rotated = test_dataset[i]['rotated']
if rotated:
image_vis = np.flipud(np.fliplr(image_vis))
pr_mask = np.flipud(np.fliplr(pr_mask))
plot_images(
SAR_intensity_0=(adjust_sar_contrast(image_vis[:, :, 0]), 'gray'),
building_mask_pr=(pr_mask, 'viridis')
)
The function which this code calls is given below:
def get_spacenet6_preprocess(config, is_test):
"""
"""
mean_path = os.path.join(
config.INPUT.MEAN_STD_DIR,
config.INPUT.IMAGE_TYPE,
'mean.npy'
)
mean = np.load(mean_path)
mean = mean[np.newaxis, np.newaxis, :]
std_path = os.path.join(
config.INPUT.MEAN_STD_DIR,
config.INPUT.IMAGE_TYPE,
'std.npy'
)
std = np.load(std_path)
std = std[np.newaxis, np.newaxis, :]
if is_test:
to_tensor = albu.Lambda(
image=functools.partial(_to_tensor)
)
else:
to_tensor = albu.Lambda(
image=functools.partial(_to_tensor),
mask=functools.partial(_to_tensor)
)
preprocess = [
albu.Lambda(
image=functools.partial(
_normalize_image,
mean=mean,
std=std
)
),
to_tensor,
]
return albu.Compose(preprocess)
I'm using Python 3 to request share statistics from Linkedin API for the last 14 months.
It works fine, when I hardcoded the epoch values in the request link:
https://api.linkedin.com/v2/organizationalEntityShareStatistics?q=organizationalEntity&organizationalEntity=urn%3Ali%3Aorganization%3AXXXXX&timeIntervals=(timeRange:(start:1596206929000,end:1632938933000),timeGranularityType:DAY)
Obviously I don't want to be changing the end values each time I make a request, so I thought I'd declare a variable:
from datetime import datetime
epochcdt = datetime.now().timestamp()
And then just use it in the link instead of the hardcoded value:
https://api.linkedin.com/v2/organizationalEntityShareStatistics?q=organizationalEntity&organizationalEntity=urn%3Ali%3Aorganization%3AXXXXX&timeIntervals=(timeRange:(start:1596206929000,end:epochcdt),timeGranularityType:DAY)
But, that doesn't work:
{'message': 'Internal Server Error', 'status': 500}
Can you please help me with this hopefully, easy to solve problem?
Whole code:
import requests
import json
from liapiauth import auth, headers
from datetime import datetime
epochcdt = (datetime.now().timestamp())*1000
def organization_info(headers):
response = requests.get('https://api.linkedin.com/v2/organizationalEntityShareStatistics?q=organizationalEntity&organizationalEntity=urn%3Ali%3Aorganization%3AXXXXX&timeIntervals=(timeRange:(start:1596206929000,end:{epochcdt}),timeGranularityType:DAY)', headers = headers)
organization_info = response.json()
return organization_info
if __name__ == '__main__':
credentials = 'credentials.json'
access_token = auth(credentials)
headers = headers(access_token)
organization_info = organization_info(headers)
with open('lishare14m.json', 'w') as outfile:
json.dump(organization_info, outfile)
print(organization_info)
I want to insert JSON type of data in PostgreSQL Database from flask
eg: {"a":[1,2,3] , "b":[1,2,3]}
One example for such data is Phone.no and Childrens, One person can have multiple ph.no and Childrens
In flask View
#app.route('/add', methods=['POST'])
def addRex():
Name = request.form[‘name’]
data = request.get_json(force=False, silent=False, cache=True)
p = Projects(name=name,data = data)
db.session.add(p)
db.session.commit()
In HTTP post method
def addData():
name = input ('Enter Name :')
data = input('Enter Data :')
headers = {'Content-type': 'application/json', 'Accept': 'text/plain'}
r = requests.post(localhost:5000/add,
data ={'name':name}, json={'data':data})
if (r.status_code == 200):print(' Added Successfully!!') else:print('Already exists!')
How can I insert such kind of data from flask into postgresql.
if Anyone can help me with my problem.
Thanks in advance
From sqlalchemy dialect, you can select JSON for Postgres. Here is an example,
from flask import Flask
from flask_sqlalchemy import SQLAlchemy
from sqlalchemy.dialects.postgresql import JSON
app = Flask(__name__)
app.config['SQLALCHEMY_DATABASE_URI'] = 'postgres://username:password#localhost:5432/db'
app.config['SQLALCHEMY_TRACK_MODIFICATIONS'] = True
db = SQLAlchemy(app)
class Person(db.Model):
person_name = db.Column(db.Text, primary_key=True)
details = db.Column(JSON)
# db.create_all() ==> for creating the db
per_1 = Person(person_name='Batman', details={"phone_no": [5, 6, 7, 8, 9], "children": {"son": [
"Dick Grayson", "Jason Todd", "Damian Wayne", "Tim Drake"], "daughter": ['Helena Wayne']}})
db.session.add(per_1)
db.session.commit()
Files to upload will be like WFSIV0101202001.318.tar.gz,WFSIV0101202001.2624.tar.gz etc.
INPUT_FILE_PATH = 'C:\Files to upload'
try:
import os
from google.cloud import storage
import sys
import pymongo
import pymongo.errors
from pymongo import MongoClient
from pymongo.errors import ConnectionFailure
except:
print("missing modules")
try:
mongo_client = MongoClient(host="xyz.com", port=27017)
Db = mongo_client['abcd']
coll = Db['shopper_journey_sitedata']
except ConnectionFailure:
print("Connection failed")
date=[]
# Thirdpartyid=[]
input_files = os.listdir(INPUT_FILE_PATH)
# looping through input files
for input_file in input_files:
x = input_file.split(".")
date.append(x[0][5:13])
tp_site_id = x[1]
# print(tp_site_id)
cur = coll.find({"third_party_site_id":tp_site_id})
for doc in cur:
print(doc)
Now i want to parameterize the find() method for every id, so that on each iteration i should get st_site_id ?
above code i tried but ist giving error as "Datas:name error"
You can do one thing
coll.find({"third_party_site_id": { $in :
[318,2624,2621,2622,102,078]}})
If Tid is an array, then you could replace 318 in your query to Tid[I]
I'm trying to scrape the live billionaire networth table here > https://www.bloomberg.com/billionaires/
This is my code so far. All I get is [] as result on the python shell.
Something has to be wrong with the "findAll", I don't think I'm using the correct tag lines.
Tried to use just "find"
from urllib.request import urlopen as uReq
from bs4 import BeautifulSoup as soup
import csv
#Open page and grab html
my_url = ('https://www.bloomberg.com/billionaires/')
uClient = uReq(my_url)
page_html = uClient.read()
uClient.close()
#HTML Parser.
page_soup = soup(page_html, 'html.parser')
table = []
#Find table.
ele_table = page_soup.findAll('div',{'class':'dvz-content'})
print(ele_table)
I'm expecting for the table to be printed out so I can get it into a CSV file.
Data is dynamically loaded. You can pull from script tag provided you supply the right headers. Regex out the required info and parse with json library. Hand this off to pandas to write to csv
from bs4 import BeautifulSoup as bs
import requests, re, json
import pandas as pd
headers = {
'user-agent': 'Mozilla/5.0',
'accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3',
'if-none-match': 'W/^\\^5dbb59e6-91b10^\\^',
'if-modified-since': 'Thu, 31 Oct 2019 22:02:14 GMT' # this may be safeguard for caching. Consider if add dynamically.
}
p = re.compile(r'window.top500 = (.*);')
r = requests.get('https://www.bloomberg.com/billionaires/', headers = headers)
data = json.loads(p.findall(r.text)[0])
df = pd.DataFrame(data)
df.to_csv(r'C:\Users\User\Desktop\Data.csv', sep=',', encoding='utf-8-sig',index = False)
Example output: