What if mitmproxy7 has no change_upstream_proxy_server? what to replace?
# mitmproxy.py
class Proxy:
def __init__(self):
pass
def request(self, flow):
if 'stackoverflow.com' in flow.request.url:
# ctx.log.info(flow.request.url)
proxy = ("127.0.0.1", 8081)
else:
proxy = ("127.0.0.1", 8082)
if flow.live:
flow.live.change_upstream_proxy_server(proxy)
test2
# mitmproxy.py
class Proxy:
def __init__(self):
pass
def request(self, flow):
if 'stackoverflow.com' in flow.request.url:
# ctx.log.info(flow.request.url)
proxy = ("127.0.0.1", 8081)
flow.live.change_upstream_proxy_server(proxy)
else:
proxy = ("127.0.0.1", 8082)
flow.live.change_upstream_proxy_server(proxy)
mitmweb.exe --mode upstream:http://127.0.0.1:8080/ -s mitmproxy.py
errinfo
AttributeError: 'NoneType' object has no attribute 'change_upstream_proxy_server'
Related
Given the following code snippet
class Config {
private val conf = ConfigFactory.load()
object Http {
val host = conf.getString("http.host")
val port = conf.getInt("http.port")
}
}
how can host and port fields of inner singleton object Http be mocked?
mockito-scala provides ReturnsDeepStubs via IdiomaticMockito
import org.mockito.stubbing.ReturnsDeepStubs
import org.mockito.{ArgumentMatchersSugar, IdiomaticMockito}
val config = mock[Config](ReturnsDeepStubs)
config.Http.host returns "www.example.com"
config.Http.port returns 80
Without deep stubbing we could do
import org.scalatest.mockito.MockitoSugar
import org.mockito.Mockito.when
val config = mock[Config]
val httpConfig = mock[config.Http.type]
when(httpConfig.host).thenReturn("www.example.com")
when(httpConfig.port).thenReturn(80)
when(config.Http).thenReturn(httpConfig)
I am rewriting some application layer code in scala from using scalaj to akka-http
in order to reduce the number of third party dependencies in the project (we already use akka for other things in the same project.) The code simply wraps common types of request to an underlying general request provided by the library
Mostly it has been fine, but I am stuck on the problem of optionally adding a proxy to a request.
Requests should either be direct to the destination or via a proxy, determined by a parameter at runtime.
In my scalaj implementation, I have the following helper class and methods
object HttpUtils {
private def request(
host: Host,
method: HttpMethod,
params: Map[String, String],
postData: Option[String],
timeout: Duration,
headers: Seq[(String, String)],
proxy: Option[ProxyConfig]
): HttpResponse[String] = {
// most general request builder. Other methods in the object fill in parameters and wrap this in a Future
val baseRequest = Http(host.url)
val proxiedRequest = addProxy(proxy, baseRequest)
val fullRequest = addPostData(postData)(proxiedRequest)
.method(method.toString)
.params(params)
.headers(headers)
.option(HttpOptions.connTimeout(timeout.toMillis.toInt))
.option(HttpOptions.readTimeout(timeout.toMillis.toInt))
fullRequest.asString // scalaj for send off request and block until response
}
// Other methods ...
private def addProxy(proxy: Option[ProxyConfig], request: HttpRequest): HttpRequest =
proxy.fold(request)((p: ProxyConfig) => request.proxy(p.host, p.port))
}
case class ProxyConfig(host: String, port: Int)
Is there a way to build a similar construct with akka-http?
Akka HTTP does have proxy support that, as of version 10.0.9, is still unstable. Keeping in mind that the API could change, you could do something like the following to handle optional proxy settings:
import java.net.InetSocketAddress
import akka.actor.ActorSystem
import akka.stream.ActorMaterializer
import akka.http.scaladsl.{ClientTransport, Http}
implicit val system = ActorSystem()
implicit val materializer = ActorMaterializer()
case class ProxyConfig(host: String, port: Int)
val proxyConfig = Option(ProxyConfig("localhost", 8888))
val clientTransport =
proxyConfig.map(p => ClientTransport.httpsProxy(InetSocketAddress.createUnresolved(p.host, p.port)))
.getOrElse(ClientTransport.TCP)
val settings = ConnectionPoolSettings(system).withTransport(clientTransport)
Http().singleRequest(HttpRequest(uri = "https://google.com"), settings = settings)
In Akka Http 10.2.0, use bindflow for a Flow[HttpRequest, HttpResponse, NotUsed] defined by a RunnableGraph with Flowshape. Insided the RunnableGraph, an Http() outgoingConnection is used to connect to the remote proxy. Some example code:
import akka.actor.typed.ActorSystem
import akka.actor.typed.scaladsl.Behaviors
import akka.http.scaladsl.Http
import akka.http.scaladsl.model.{HttpRequest, HttpResponse}
import akka.stream._
import akka.stream.scaladsl.{Broadcast, Flow, GraphDSL, Merge}
import scala.concurrent.ExecutionContextExecutor
import scala.concurrent.duration.DurationInt
import scala.io.StdIn
import scala.util.{Failure, Success}
object Main {
def main(args: Array[String]) {
implicit val system: ActorSystem[Nothing] = ActorSystem(Behaviors.empty, "testproxy")
implicit val executionContext: ExecutionContextExecutor = system.executionContext
system.log.info("TestAkkaHttpProxy Main started...")
val remoteHost = "xxx.xxx.xxx.x"
val remotePort = 8000
val proxyHost = "0.0.0.0"
val proxyPort = 8080
val gateway = Flow.fromGraph(GraphDSL.create() { implicit b =>
import GraphDSL.Implicits._
// Broadcast for flow input
val broadcaster = b.add(Broadcast[HttpRequest](1))
// Merge for flow output
val responseMerge = b.add(Merge[HttpResponse](1))
// outgoing client for remote proxy
val remote = Http().outgoingConnection(remoteHost, remotePort)
// filter out header that creates Akka Http warning
val requestConvert = Flow[HttpRequest]
.map(req => { req.mapHeaders(headers => headers.filter(h => h.isNot("timeout-access")))
})
// connect graph
broadcaster.out(0) ~> requestConvert ~> remote ~> responseMerge
// expose ports
FlowShape(broadcaster.in, responseMerge.out)
})
// Akka Http server that binds to Flow (for remote proxy)
Http().newServerAt(proxyHost, proxyPort).bindFlow(gateway)
.onComplete({
case Success(binding) ⇒
println(s"Server is listening on 0.0.0.0:8080")
binding.addToCoordinatedShutdown(hardTerminationDeadline = 10.seconds)
case Failure(e) ⇒
println(s"Binding failed with ${e.getMessage}")
system.terminate()
})
system.log.info("Press RETURN to stop...")
StdIn.readLine()
system.terminate()
}
}
i have this Scala code in order to connect to some FTP server:
import java.io.{File, FileOutputStream, InputStream}
import org.apache.commons.net.ftp.{FTPClient,FTPFile}
import scala.util.Try
final class FTP() {
private val client = new FTPClient
def login(username: String, password: String): Try[Boolean] = Try {
client.login(username, password)
}
def connect(host: String): Try[Unit] = Try {
client.connect(host)
client.enterLocalPassiveMode()
}
def connected: Boolean = client.isConnected
def disconnect(): Unit = client.disconnect()
def canConnect(host: String): Boolean = {
client.connect(host)
val connectionWasEstablished = connected
client.disconnect()
connectionWasEstablished
}
def listFiles(dir: Option[String] = None): List[FTPFile] =
dir.fold(client.listFiles)(client.listFiles).toList
def connectWithAuth(host: String,
username: String = "anonymous",
password: String = "") : Try[Boolean] = {
for {
connection <- connect(host)
login <- login(username, password)
} yield login
}
def cd(path: String): Boolean =
client.changeWorkingDirectory(path)
def filesInCurrentDirectory: Seq[String] =
listFiles().map(_.getName)
def downloadFileStream(remote: String): InputStream = {
val stream = client.retrieveFileStream(remote)
client.completePendingCommand()
stream
}
def downloadFile(remote: String): Boolean = {
val os = new FileOutputStream(new File(remote))
client.retrieveFile(remote, os)
}
def uploadFile(remote: String, input: InputStream): Boolean =
client.storeFile(remote, input)
}// end of final class
val myFTP = new FTP
val myBool : Try[Boolean] = myFTP.connectWithAuth("MY_FTP_SERVER","USER","PASS")
println("Can connect? " + myBool)
When I run this code in a local spark-shell, I have this:
scala> val myFTP = new FTP
myFTP: FTP = $iwC$$iwC$FTP#1077598b
scala>
scala> val myBool : Try[Boolean] = myFTP.connectWithAuth("MY_FTP_SERVER","USER","PASS")
myBool: scala.util.Try[Boolean] = Failure(java.net.UnknownHostException: MY_FTP_SERVER: nodename nor servname provided, or not known)
scala> println("Can connect? " + myBool)
Can connect? Failure(java.net.UnknownHostException: MY_FTP_SERVER: nodename nor servname provided, or not known)
If I export http_proxy variable in a shell, I can reach to the ftp server via curl, so, how can I programmatically set http_proxy variable using Scala?
MacBook-Pro-de-Alonso:spark-1.5.0-bin-hadoop2.6 aisidoro$ export http_proxy=socks5://MY_PROXY:1080
MacBook-Pro-de-Alonso:spark-1.5.0-bin-hadoop2.6 aisidoro$ curl MY_FTP_SERVER
<!DOCTYPE HTML PUBLIC "-//IETF//DTD HTML 2.0//EN">
<html><head>
<title>301 Moved Permanently</title>
</head><body>
<h1>Moved Permanently</h1>
<p>The document has moved here.</p>
<hr>
<address>Apache/2.2.15 (CentOS) Server at MY_FTP_SERVER Port 80</address>
</body></html>
Thank you very much.
UPDATE 22 Dic 2016
I have tried to do this in the same spark-shell that I run the FTP class code:
scala> System.setProperty("http_proxy","socks5://MY_PROXY_SERVER:1080")
res0: String = null
scala> System.getProperty("http_proxy")
res1: String = socks5://MY_PROXY_SERVER:1080
with the same result, I cannot reach programmatically to this ftp server through a proxy server...
I also have tried to export JAVA_OPTS before running the spark-shell in the same terminal session, with the same result...
export JAVA_OPTS="Dhttp.proxyHost=socks5://MY_PROXY_HOST -Dhttp.proxyPort=1080"
While the http_proxy environment variables are commonly used by curl and various other utilities, java has its own way of specifying proxy settings. Since you're using a SOCKS proxy, try setting the socks proxy properties, e.g. if using command line/environment variables:
-DsocksProxyHost=MY_PROXY_HOST -DsocksProxyPort=1080
If using the spark shell, you may want to look at for example this StackOverflow question regarding passing these through.
The Apache Commons FtpClient also doesn't do anything to explicitly look at the environment variables, and uses socket.connect to connect to the the FTP server.
However, there is a FTPHTTPClient class that might be handy if you wanted to tunnel FTP traffic via a HTTP proxy, see the example for details, but basically it's:
ftp = new FTPHTTPClient(proxyHost, proxyPort, proxyUser, proxyPassword);
I have two spray http clients, such as the following:
val pipelineFoo: HttpRequest => Future[Foo] = (
sendReceive
~> unmarshal[Message.Foo])
val pipelineBar: HttpRequest => Future[Bar] = (
sendReceive
~> unmarshal[Message.Bar])
def execFoo(h: String, p: Int): Future[Foo] = {
val uri = Uri.from(scheme = "http", host = h, port = p, path = "/foo")
pipelineFoo(Get(uri))
}
def execBar(h: String, p: Int): Future[Bar] = {
val uri = Uri.from(scheme = "http", host = h, port = p, path = "/bar")
pipelineBar(Get(uri))
}
I would like have the foo request retry several times with a long timeout, and have the bar request not retry and have a short timeout (say 1 second). How can I achieve this in spray (sorry if this is somewhere in the documentation, but I've been unable to find it -- I've only found some documentation on setting such config parameters globally).
This shouldn't be too difficult. sendReceive can actually take more parameters. For example, here is the signature for one of the alternatives:
def sendReceive(transport: ActorRef)(implicit ec: ExecutionContext, futureTimeout: Timeout): SendReceive
I use this myself for similar scenarios where I have to have a bigger number of retries and longer timeouts when I hit an external service vs hitting one of our internal ones.
Here's an example of my pipeline that I use:
lazy val pipeline: HttpRequest => Future[HttpResponse] = (
addCredentials(BasicHttpCredentials(clientConnection.credentials._1, clientConnection.credentials._2))
~> addHeader(`User-Agent`(ProductVersion("<YOUR NAME HERE>", "<YOUR VERSION HERE>", "http://github.com/<WHEREVER YOUR PROJECT IS>"), ProductVersion("spray-client", "1.3.1", "http://spray.io")))
~> logRequest(log)
~> sendReceive(clientConnection.connection)(clientConnection.context, clientConnection.timeout)
~> decode(Deflate)
~> decode(Gzip)
)
The clientConnection is nothing special. It's just a case class that I made that can be filled in manually via code or maybe some config in your application.conf
2 years latter, but maybe worth for other people.
We had the same need and we based our solution on a copy/paste of Spray connector files.
import akka.actor.{ActorRef, ActorSystem}
import akka.io.IO
import akka.pattern.ask
import com.typesafe.config.Config
import spray.can.Http
import spray.can.Http.HostConnectorSetup
import spray.can.client.HostConnectorSettings
import spray.client.pipelining.sendReceive
import spray.http.Uri.Host
import spray.http.{HttpRequest, HttpResponse, Uri}
import scala.concurrent.duration._
import scala.concurrent.{ExecutionContextExecutor, Future}
case class HttpCustomSettings(
requestTimeout: Duration,
maxRetries: Int,
maxConnections: Int
)
/**
* Implement a new HTTP client on top of akka IO and spray HTTP
* to provide a way for caller to set client parameters on request basis instead
* of globally in application.conf
*
* This client defaults all its configuration with the one set in spray.conf
* see spray.can.client and spray.can.host-connector
* But you can override some of them on demand
* - maxRetries
* - requestTimeout
* - maxConnections
*/
class HttpClient(actorSystem: ActorSystem, config: Config) {
private implicit val context: ActorSystem = actorSystem
private implicit val dispatcher: ExecutionContextExecutor = actorSystem.dispatcher
private val HTTP = "http"
private val HTTPS = "https"
private val defaultSettings: HostConnectorSettings =
HostConnectorSettings.fromSubConfig(config.getConfig("spray.can"))
//not configurable since this timeout has little to no use practically
//this timeout DOES NOT kill the open connection
//http://kamon.io/teamblog/2014/11/02/understanding-spray-client-timeout-settings/
private implicit val clientFutureTimeout: akka.util.Timeout = 5.seconds
def send(
request: HttpRequest,
customSettings: Option[HttpCustomSettings] = None
): Future[HttpResponse] = {
val pipeline: Future[HttpRequest ⇒ Future[HttpResponse]] =
pipelineForUri(request.uri, customSettings)
pipeline.flatMap(send ⇒ send(request))
}
/**
* To understand more this method
* #see http://kamon.io/assets/img/diagrams/spray-client-actors.png
* #see [[spray.can.HttpManager]]
* #see [[spray.can.client.HttpHostConnector]]
* #see [[spray.can.Http]]
*/
private def pipelineForUri(
uri: Uri,
customSettings: Option[HttpCustomSettings]
): Future[HttpRequest ⇒ Future[HttpResponse]] = {
for {
Http.HostConnectorInfo(connector, _) ← IO(Http) ? connectorSetup(uri, customSettings)
} yield sendReceive(connector)
}
private def connectorSetup(
uri: Uri,
customSettings: Option[HttpCustomSettings]
): HostConnectorSetup = {
require(
uri.scheme == HTTP || uri.scheme == HTTPS,
s"Not a valid $HTTP URI scheme: '${uri.scheme}' in '$uri'. (Did you forget $HTTP:// ?)"
)
val connector: HostConnectorSetup = HostConnectorSetup(
uri.authority.host.toString,
uri.effectivePort,
sslEncryption = uri.scheme == HTTPS
)
customSettings match {
case Some(custom) ⇒ connector.copy(settings = Option(mapCustomSettings(defaultSettings, custom)))
case None ⇒ connector.copy(settings = Option(defaultSettings))
}
}
private def mapCustomSettings(
settings: HostConnectorSettings,
customSettings: HttpCustomSettings
): HostConnectorSettings = {
settings.copy(
maxRetries = customSettings.maxRetries,
maxConnections = customSettings.maxConnections,
connectionSettings = settings.connectionSettings.copy(requestTimeout = customSettings.requestTimeout)
)
}
}
I've just run into an issue running unittests on my flask app after I had roughly 100 unittests. All unittests will pass, but when run all at once they will fail with the following error:
OperationalError: (OperationalError) FATAL: remaining connection slots are reserved for non-replication superuser connections
Everything is running in a virtualbox/vagrant/ubuntu12.04 instance on local machine. My postgres max_connections is set to 100 so I'm assuming that the connections aren't closing and after running 100 tests I use up all the available ones.
This person Flask unit tests with SQLAlchemy and PostgreSQL exhausts db connections looks like they are having the same exact problem. Mike/Zzzeek (sqlalchemy dev) even responded to it saying that something may be happening in create_app() so I've included that as well below.
Does this mean I'm not closing my connections somewhere? All of these errors are triggered by db.create_all() in my setUp() method of my unittest.
# test.py
class TestCase(DataMixin, Base):
"""Base test class"""
def create_app(self):
return create_app(TestConfig())
def setUp(self):
db.create_all()
def tearDown(self):
db.session.remove()
db.drop_all()
# app.py
def create_app(config=None):
app = Flask(__name__)
# Config
app.config.from_object(BaseConfig())
if config is not None:
app.config.from_object(config)
# Extensions
db.init_app(app)
mail.init_app(app)
bcrypt.init_app(app)
# Blueprints
app.register_blueprint(core_blueprint, url_prefix='/')
app.register_blueprint(accounts_blueprint, url_prefix='/account')
app.register_blueprint(admin_blueprint, url_prefix='/admin')
app.register_blueprint(cart_blueprint, url_prefix='/cart')
# Login Manager
login_manager.setup_app(app, add_context_processor=True)
login_manager.login_view = "accounts.login"
login_manager.user_callback = load_user
# Templates
app.jinja_env.globals['is_admin'] = is_admin
app.jinja_env.globals['is_staff'] = is_staff
#app.context_processor
def inject_cart():
cart = count = None
if current_user.is_authenticated():
cart = current_user.get_cart()
return dict(cart=cart)
# Error Handling
#app.errorhandler(404)
def page_not_found(error):
return render_template('404.html'), 404
return app
UPDATE: Tested and fixed
Instead of making a new connection and re-creating your database every time (slow), you can use subsessions and do a rollback after each test.
The connection are reused, so this also fix the problem you're having.
class TestCase(Base):
#classmethod
def setUpClass(cls):
cls.app = create_app(MyConfig())
cls.client = cls.app.test_client()
cls._ctx = cls.app.test_request_context()
cls._ctx.push()
db.create_all()
#classmethod
def tearDownClass(cls):
db.session.remove()
db.drop_all()
db.get_engine(cls.app).dispose()
def setUp(self):
self._ctx = self.app.test_request_context()
self._ctx.push()
db.session.begin(subtransactions=True)
def tearDown(self):
db.session.rollback()
db.session.close()
self._ctx.pop()
If you need to also make an instance of the application for each test, just add it to the setUp method but leave it also in setUpClass.
Full test example below requires flask_sqlalchemy and psycopg2. Create a test database named "test" and set its connection limit to 15.
from flask import Flask
from flask.ext.sqlalchemy import SQLAlchemy
from unittest import TestCase as Base
db = SQLAlchemy()
def create_app(config=None):
app = Flask(__name__)
app.config.from_object(config)
db.init_app(app)
return app
class MyConfig(object):
SQLALCHEMY_DATABASE_URI = "postgresql://localhost/test"
TESTING = True
class TestCase(Base):
#classmethod
def setUpClass(cls):
cls.app = create_app(MyConfig())
cls.client = cls.app.test_client()
cls._ctx = cls.app.test_request_context()
cls._ctx.push()
db.create_all()
#classmethod
def tearDownClass(cls):
db.session.remove()
db.drop_all()
def setUp(self):
self._ctx = self.app.test_request_context()
self._ctx.push()
db.session.begin(subtransactions=True)
def tearDown(self):
db.session.rollback()
db.session.close()
self._ctx.pop()
class TestModel(TestCase):
def test_01(self):
pass
def test_02(self):
pass
def test_03(self):
pass
def test_04(self):
pass
def test_05(self):
pass
def test_06(self):
pass
def test_07(self):
pass
def test_08(self):
pass
def test_09(self):
pass
def test_10(self):
pass
def test_11(self):
pass
def test_12(self):
pass
def test_13(self):
pass
def test_14(self):
pass
def test_15(self):
pass
def test_16(self):
pass
if __name__ == "__main__":
import unittest
unittest.main()
I found the answer here -- https://stackoverflow.com/a/17998485/1870623 and a great explination here -- https://stackoverflow.com/a/16390645/1870623
The solution is to add db.get_engine(self.app).dispose() to the tearDown()
class TestCase(Base):
def setUp(self):
db.create_all()
def tearDown(self):
db.session.remove()
db.drop_all()
db.get_engine(self.app).dispose() # This