I'm attempting to deploy an oracle service bus project to my locally hosted weblogic 12c server in an internet-restricted VM but the tools are indicating my JAR file isn't a valid application file. What is the proper way of building and deploying OSB projects to the weblogic host?
I've attempted to build use both the configjar utility to create jar files as well as exported directly from JDeveloper to a jar file. I've also attempted to use the ant task jwsc to build the osb project, but haven't been successful.
I've attempted to deploy via the deploy() WLST command, the wldeploy ANT task, and the the wldeploy utility tool but they run into an error and quit. I am able to take the .jar file and manually upload it through the service bus console without issue, though.
connect(username, password, adminUrl)
deploy(deploymentName,deploymentFile,targets=deploymentTarget)
startApplication(deploymentName)
I was expecting that the above sample code would deploy the application successfully, but instead the following error code is returned every time:
Deployment Message : weblogic.management.DeploymentException:
[J2EE:160177]The application at
"C:\jdeveloper\mywork\CommonServicesOSB\CrmConnections\test3.jar" was
not recognized as a valid application type. If this is an EAR file,
please ensure the META-INF/application.xml exists. EJB-JARs should
have a META-INF/ejb-jar.xml or corresponding annotations exist. If
this is an exploded WAR, the name of directory must be end with
".war". RARs require a META-INF/ra.xml. A JMS deployment should be an
XML file whose name ends with "-jms.xml". A JDBC deployment should be
an XML file whose name ends with "-jdbc.xml". For other application
types, consult the WebLogic Server documentation.
I'm guessing that I'm missing a crucial file or step, but the documentation I can find hasn't made this any clearer. Does anyone know how this is supposed to work?
In Weblogic 11c i used the following script.
Command to run script:
./oracle_common/common/bin/wlst.sh script.py import.properties path_jar.jar
from java.util import HashMap
from java.util import HashSet
from java.util import ArrayList
from java.io import FileInputStream
from com.bea.wli.sb.util import Refs
from com.bea.wli.config.customization import Customization
from com.bea.wli.sb.management.importexport import ALSBImportOperation
import sys
#=======================================================================================
# Entry function to deploy project configuration and resources
# into a ALSB domain
#=======================================================================================
def importToALSBDomain(importConfigFile, importJarPath):
try:
SessionMBean = None
print 'Loading Deployment config from :', importConfigFile
exportConfigProp = loadProps(importConfigFile)
adminUrl = exportConfigProp.get("adminUrl")
importUser = exportConfigProp.get("importUser")
importPassword = exportConfigProp.get("importPassword")
#importJar = exportConfigProp.get("importJar")
customFile = exportConfigProp.get("customizationFile")
passphrase = exportConfigProp.get("passphrase")
project = exportConfigProp.get("project")
connectToServer(importUser, importPassword, adminUrl)
print 'Attempting to import :', importJarPath, "on ALSB Admin Server listening on :", adminUrl
theBytes = readBinaryFile(importJarPath)
print 'Read file', importJarPath
sessionName = createSessionName()
print 'Created session', sessionName
SessionMBean = getSessionManagementMBean(sessionName)
print 'SessionMBean started session'
ALSBConfigurationMBean = findService(String("ALSBConfiguration.").concat(sessionName), "com.bea.wli.sb.management.configuration.ALSBConfigurationMBean")
print "ALSBConfiguration MBean found", ALSBConfigurationMBean
ALSBConfigurationMBean.uploadJarFile(theBytes)
print 'Jar Uploaded'
if project == None:
print 'No project specified, additive deployment performed'
alsbJarInfo = ALSBConfigurationMBean.getImportJarInfo()
alsbImportPlan = alsbJarInfo.getDefaultImportPlan()
alsbImportPlan.setPassphrase(passphrase)
alsbImportPlan.setPreserveExistingEnvValues(true)
importResult = ALSBConfigurationMBean.importUploaded(alsbImportPlan)
SessionMBean.activateSession(sessionName, "Complete test import with customization using wlst")
else:
print 'ALSB project', project, 'will get overlaid'
alsbJarInfo = ALSBConfigurationMBean.getImportJarInfo()
alsbImportPlan = alsbJarInfo.getDefaultImportPlan()
alsbImportPlan.setPassphrase(passphrase)
operationMap=HashMap()
operationMap = alsbImportPlan.getOperations()
print
print 'Default importPlan'
printOpMap(operationMap)
set = operationMap.entrySet()
alsbImportPlan.setPreserveExistingEnvValues(true)
#boolean
abort = false
#list of created ref
createdRef = ArrayList()
for entry in set:
ref = entry.getKey()
op = entry.getValue()
#set different logic based on the resource type
type = ref.getTypeId
if type == Refs.SERVICE_ACCOUNT_TYPE or type == Refs.SERVICE_PROVIDER_TYPE:
if op.getOperation() == ALSBImportOperation.Operation.Create:
print 'Unable to import a service account or a service provider on a target system', ref
abort = true
elif op.getOperation() == ALSBImportOperation.Operation.Create:
#keep the list of created resources
createdRef.add(ref)
if abort == true :
print 'This jar must be imported manually to resolve the service account and service provider dependencies'
SessionMBean.discardSession(sessionName)
raise
print
print 'Modified importPlan'
printOpMap(operationMap)
importResult = ALSBConfigurationMBean.importUploaded(alsbImportPlan)
printDiagMap(importResult.getImportDiagnostics())
if importResult.getFailed().isEmpty() == false:
print 'One or more resources could not be imported properly'
raise
#customize if a customization file is specified
#affects only the created resources
if customFile != None :
print 'Loading customization File', customFile
print 'Customization applied to the created resources only', createdRef
iStream = FileInputStream(customFile)
customizationList = Customization.fromXML(iStream)
filteredCustomizationList = ArrayList()
setRef = HashSet(createdRef)
# apply a filter to all the customizations to narrow the target to the created resources
for customization in customizationList:
print customization
newcustomization = customization.clone(setRef)
filteredCustomizationList.add(newcustomization)
ALSBConfigurationMBean.customize(filteredCustomizationList)
SessionMBean.activateSession(sessionName, "Complete test import with customization using wlst")
print "Deployment of : " + importJarPath + " successful"
except:
print "Unexpected error:", sys.exc_info()[0]
if SessionMBean != None:
SessionMBean.discardSession(sessionName)
raise
#=======================================================================================
# Utility function to print the list of operations
#=======================================================================================
def printOpMap(map):
set = map.entrySet()
for entry in set:
op = entry.getValue()
print op.getOperation(),
ref = entry.getKey()
print ref
print
#=======================================================================================
# Utility function to print the diagnostics
#=======================================================================================
def printDiagMap(map):
set = map.entrySet()
for entry in set:
diag = entry.getValue().toString()
print diag
print
#=======================================================================================
# Utility function to load properties from a config file
#=======================================================================================
def loadProps(configPropFile):
propInputStream = FileInputStream(configPropFile)
configProps = Properties()
configProps.load(propInputStream)
return configProps
#=======================================================================================
# Connect to the Admin Server
#=======================================================================================
def connectToServer(username, password, url):
connect(username, password, url)
domainRuntime()
#=======================================================================================
# Utility function to read a binary file
#=======================================================================================
def readBinaryFile(fileName):
file = open(fileName, 'rb')
bytes = file.read()
return bytes
#=======================================================================================
# Utility function to create an arbitrary session name
#=======================================================================================
def createSessionName():
sessionName = String("SessionScript"+Long(System.currentTimeMillis()).toString())
return sessionName
#=======================================================================================
# Utility function to load a session MBeans
#=======================================================================================
def getSessionManagementMBean(sessionName):
SessionMBean = findService("SessionManagement", "com.bea.wli.sb.management.configuration.SessionManagementMBean")
SessionMBean.createSession(sessionName)
return SessionMBean
# IMPORT script init
try:
# import the service bus configuration
# argv[1] is the export config properties file
importToALSBDomain(sys.argv[1], sys.argv[2])
except:
print "Unexpected error: ", sys.exc_info()[0]
dumpStack()
raise
and with the following import.properties file:
##################################################################
# OSB Admin Configuration #
##################################################################
adminUrl=t3://localhost:7001
importUser=weblogic
importPassword=weblogic89
Related
We are using py.test 2.8.7 and I have the below method which creates a separate log file for every test-case. However this does not handle unhandled Exceptions. So if a code snippet throws an Exception instead of failing with an assert, the stack-trace of the Exception is not logged into the separate file. Can someone please help me in how I could capture these Exceptions?
def remove_special_chars(input):
"""
Replaces all special characters which ideally shout not be included in the name of a file
Such characters will be replaced with a dot so we know there was something useful there
"""
for special_ch in ["/", "\\", "<", ">", "|", "&", ":", "*", "?", "\"", "'"]:
input = input.replace(special_ch, ".")
return input
def assemble_test_fqn(node):
"""
Assembles a fully-qualified name for our test-case which will be used as its test log file name
"""
current_node = node
result = ""
while current_node is not None:
if current_node.name == "()":
current_node = current_node.parent
continue
if result != "":
result = "." + result
result = current_node.name + result
current_node = current_node.parent
return remove_special_chars(result)
# This fixture creates a logger per test-case
#pytest.yield_fixture(scope="function", autouse=True)
def set_log_file_per_method(request):
"""
Creates a separate file logging handler for each test method
"""
# Assembling the location of the log folder
test_log_dir = "%s/all_test_logs" % (request.config.getoption("--output-dir"))
# Creating the log folder if it does not exist
if not os.path.exists(test_log_dir):
os.makedirs(test_log_dir)
# Adding a file handler
test_log_file = "%s/%s.log" % (test_log_dir, assemble_test_fqn(request.node))
file_handler = logging.FileHandler(filename=test_log_file, mode="w")
file_handler.setLevel("INFO")
log_format = request.config.getoption("--log-format")
log_formatter = logging.Formatter(log_format)
file_handler.setFormatter(log_formatter)
logging.getLogger('').addHandler(file_handler)
yield
# After the test finished, we remove the file handler
file_handler.close()
logging.getLogger('').removeHandler(file_handler)
I have ended-up with a custom plugin:
import io
import os
import pytest
def remove_special_chars(text):
"""
Replaces all special characters which ideally shout not be included in the name of a file
Such characters will be replaced with a dot so we know there was something useful there
"""
for special_ch in ["/", "\\", "<", ">", "|", "&", ":", "*", "?", "\"", "'"]:
text = text.replace(special_ch, ".")
return text
def assemble_test_fqn(node):
"""
Assembles a fully-qualified name for our test-case which will be used as its test log file name
The result will also include the potential path of the log file as the parents are appended to the fqn with a /
"""
current_node = node
result = ""
while current_node is not None:
if current_node.name == "()":
current_node = current_node.parent
continue
if result != "":
result = "/" + result
result = remove_special_chars(current_node.name) + result
current_node = current_node.parent
return result
def as_unicode(text):
"""
Encodes a text into unicode
If it's already unicode, we do not touch it
"""
if isinstance(text, unicode):
return text
else:
return unicode(str(text))
class TestReport:
"""
Holds a test-report
"""
def __init__(self, fqn):
self._fqn = fqn
self._errors = []
self._sections = []
def add_error(self, error):
"""
Adds an error (either an Exception or an assertion error) to the list of errors
"""
self._errors.append(error)
def add_sections(self, sections):
"""
Adds captured sections to our internal list of sections
Since tests can have multiple phases (setup, call, teardown) this will be invoked for all phases
If for a newer phase we already captured a section, we override it in our already existing internal list
"""
interim = []
for current_section in self._sections:
section_to_add = current_section
# If the current section we already have is also present in the input parameter,
# we override our existing section with the one from the input as that's newer
for index, input_section in enumerate(sections):
if current_section[0] == input_section[0]:
section_to_add = input_section
sections.pop(index)
break
interim.append(section_to_add)
# Adding the new sections from the input parameter to our internal list
for input_section in sections:
interim.append(input_section)
# And finally overriding our internal list of sections
self._sections = interim
def save_to_file(self, log_folder):
"""
Saves the current report to a log file
"""
# Adding a file handler
test_log_file = "%s/%s.log" % (log_folder, self._fqn)
# Creating the log folder if it does not exist
if not os.path.exists(os.path.dirname(test_log_file)):
os.makedirs(os.path.dirname(test_log_file))
# Saving the report to the given log file
with io.open(test_log_file, 'w', encoding='UTF-8') as f:
for error in self._errors:
f.write(as_unicode(error))
f.write(u"\n\n")
for index, section in enumerate(self._sections):
f.write(as_unicode(section[0]))
f.write(u":\n")
f.write((u"=" * (len(section[0]) + 1)) + u"\n")
f.write(as_unicode(section[1]))
if index < len(self._sections) - 1:
f.write(u"\n")
class ReportGenerator:
"""
A py.test plugin which collects the test-reports and saves them to a separate file per test
"""
def __init__(self, output_dir):
self._reports = {}
self._output_dir = output_dir
#pytest.hookimpl(tryfirst=True, hookwrapper=True)
def pytest_runtest_makereport(self, item, call):
outcome = yield
# Generating the fully-qualified name of the underlying test
fqn = assemble_test_fqn(item)
# Getting the already existing report for the given test from our internal dict or creating a new one if it's not already present
# We need to do this as this method will be invoked for each phase (setup, call, teardown)
if fqn not in self._reports:
report = TestReport(fqn)
self._reports.update({fqn: report})
else:
report = self._reports[fqn]
result = outcome.result
# Appending the sections for the current phase to the test-report
report.add_sections(result.sections)
# If we have an error, we add that as well to the test-report
if hasattr(result, "longrepr") and result.longrepr is not None:
error = result.longrepr
error_text = ""
if isinstance(error, str) or isinstance(error, unicode):
error_text = as_unicode(error)
elif isinstance(error, tuple):
error_text = u"\n".join([as_unicode(e) for e in error])
elif hasattr(error, "reprcrash") and hasattr(error, "reprtraceback"):
if error.reprcrash is not None:
error_text += str(error.reprcrash)
if error.reprtraceback is not None:
if error_text != "":
error_text += "\n\n"
error_text += str(error.reprtraceback)
else:
error_text = as_unicode(error)
report.add_error(error_text)
# Finally saving the report
# We need to do this for all phases as we don't know if and when a test would fail
# This will essentially override the previous log file for a test if we are in a newer phase
report.save_to_file("%s/all_test_logs" % self._output_dir)
def pytest_configure(config):
config._report_generator = ReportGenerator("result")
config.pluginmanager.register(config._report_generator)
I am trying to add a procedure to pop-up a modal dialog inside a plug-in.
Its purpose is to query a response at designated steps within the control-flow of the plug-in (not just acquire parameters at its start).
I have tried using gtk - I get a dialog but it is asynchronous - the plugin continues execution. It needs to operate as a synchronous function.
I have tried registering a plugin in order to take advantage of the gimpfu start-up dialogue for same. By itself, it works; it shows up in the procedural db when queried. But I never seem to be able to actually invoke it from within another plug-in - its either an execution error or wrong number of arguments no matter how many permutations I try.
[Reason behind all of this nonsense: I have written a lot of extension Python scripts for PaintShopPro. I have written a App package (with App.Do, App.Constants, Environment and the like that lets me begin to port those scripts to GIMP -- yes it is perverse, and yes sometimes the code just has to be rewritten, but for a lot of what I actual use in the PSP.API it is sufficient.
However, debugging and writing the module rhymes with witch. So. I am trying to add emulation of psp's "SetExecutionMode" (ie interactive). If
set, the intended behavior is that the App.Do() method will "pause" after/before it runs the applicable psp emulation code by popping up a simple message dialog.]
A simple modal dialogue within a gimp python-fu plug-in can be implemented via gtk's Dialog interface, specifically gtk.MessageDialog.
A generic dialog can be created via
queryDialogue = gtk.MessageDialog(None, gtk.DIALOG_DESTROY_WITH_PARENT \
gtk.MESSAGE_QUESTION, \
gtk.BUTTONS_OK_CANCEL, "")
Once the dialog has been shown,
a synchronous response may be obtained from it
queryDialogue.show()
response = queryDialogue.run()
queryDialogue.hide()
The above assumes that the dialog is not created and thence destroyed after each use.
In the use case (mentioned in the question) of a modal dialog to manage single stepping through a pspScript in gimp via an App emulator package, the dialogue message contents need to be customized for each use. [Hence, the "" for the message argument in the Constructor. [more below]]
In addition, the emulator must be able to accept a [cancel] response to 'get out of Dodge' - ie quit the entire plug-in (gracefully). I could not find a gimpfu interface for the latter, (and do not want to kill the app entirely via gimp.exit()). Hence, this is accomplished by raising a custom Exception class [appTerminate] within the App pkg and catching the exception in the outer-most scope of the plugin. When caught, then, the plug-in returns (exits).[App.Do() can not return a value to indicate continue/exit/etc, because the pspScripts are to be included verbatim.]
The following is an abbreviated skeleton of the solution -
a plug-in incorporating (in part) a pspScript
the App.py pkg supplying the environment and App.Do() to support the pspScript
a Map.py pkg supporting how pspScripts use dot-notation for parameters
App.py demonstrates creation, customization and use of a modal dialog - App.doContinue() displays the dialogue illustrating how it can be customized on each use.
App._parse() parses the pspScript (excerpt showing how it determines to start/stop single-step via the dialogue)
App._exec() implements the pspScript commands (excerpt showing how it creates the dialogue, identifies the message widget for later customization, and starts/stops its use)
# App.py (abbreviated)
#
import gimp
import gtk
import Map # see https://stackoverflow.com/questions/2352181/how-to- use-a-dot-to-access-members-of-dictionary
from Map import *
pdb = gimp.pdb
isDialogueAvailable = False
queryDialogue = None
queryMessage = None
Environment = Map({'executionMode' : 1 })
_AutoActionMode = Map({'Match' : 0})
_ExecutionMode = Map({'Default' : 0}, Silent=1, Interactive=2)
Constants = Map({'AutoActionMode' : _AutoActionMode}, ExecutionMode=_ExecutionMode ) # etc...
class appTerminate(Exception): pass
def Do(eNvironment, procedureName, options = {}):
global appTerminate
img = gimp.image_list()[0]
lyr = pdb.gimp_image_get_active_layer(img)
parsed = _parse(img, lyr, procedureName, options)
if eNvironment.executionMode == Constants.ExecutionMode.Interactive:
resp = doContinue(procedureName, parsed.detail)
if resp == -5: # OK
print procedureName # log to stdout
if parsed.valid:
if parsed.isvalid:
_exec(img, lyr, procedureName, options, parsed, eNvironment)
else:
print "invalid args"
else:
print "invalid procedure"
elif resp == -6: # CANCEL
raise appTerminate, "script cancelled"
pass # terminate plugin
else:
print procedureName + " skipped"
pass # skip execution, continue
else:
_exec(img, lyr, procedureName, options, parsed, eNvironment)
return
def doContinue(procedureName, details):
global queryMessage, querySkip, queryDialogue
# - customize the dialog -
if details == "":
msg = "About to execute procedure \n "+procedureName+ "\n\nContinue?"
else:
msg = "About to execute procedure \n "+procedureName+ "\n\nDetails - \n" + details +"\n\nContinue?"
queryMessage.set_text(msg)
queryDialogue.show()
resp = queryDialogue.run() # get modal response
queryDialogue.hide()
return resp
def _parse(img, lyr, procedureName, options):
# validate and interpret App.Do options' semantics vz gimp
if procedureName == "Selection":
isValid=True
# ...
# parsed = Map({'valid' : True}, isvalid=True, start=Start, width=Width, height=Height, channelOP=ChannelOP ...
# /Selection
# ...
elif procedureName == "SetExecutionMode":
generalOptions = options['GeneralSettings']
newMode = generalOptions['ExecutionMode']
if newMode == Constants.ExecutionMode.Interactive:
msg = "set mode interactive/single-step"
else:
msg = "set mode silent/run"
parsed = Map({'valid' : True}, isvalid=True, detail=msg, mode=newMode)
# /SetExecutionMode
else:
parsed = Map({'valid' : False})
return parsed
def _exec(img, lyr, procedureName, options, o, eNvironment):
global isDialogueAvailable, queryMessage, queryDialogue
#
try:
# -------------------------------------------------------------------------------------------------------------------
if procedureName == "Selection":
# pdb.gimp_rect_select(img, o.start[0], o.start[1], o.width, o.height, o.channelOP, ...
# /Selection
# ...
elif procedureName == "SetExecutionMode":
generalOptions = options['GeneralSettings']
eNvironment.executionMode = generalOptions['ExecutionMode']
if eNvironment.executionMode == Constants.ExecutionMode.Interactive:
if isDialogueAvailable:
queryDialogue.destroy() # then clean-up and refresh
isDialogueAvailable = True
queryDialogue = gtk.MessageDialog(None, gtk.DIALOG_DESTROY_WITH_PARENT, gtk.MESSAGE_QUESTION, gtk.BUTTONS_OK_CANCEL, "")
queryDialogue.set_title("psp/APP.Do Emulator")
queryDialogue.set_size_request(450, 180)
aqdContent = queryDialogue.children()[0]
aqdHeader = aqdContent.children()[0]
aqdMsgBox = aqdHeader.children()[1]
aqdMessage = aqdMsgBox.children()[0]
queryMessage = aqdMessage
else:
if isDialogueAvailable:
queryDialogue.destroy()
isDialogueAvailable = False
# /SetExecutionMode
else: # should not get here (should have been screened by parse)
raise AssertionError, "unimplemented PSP procedure: " + procedureName
except:
raise AssertionError, "App.Do("+procedureName+") generated an exception:\n" + sys.exc_info()
return
A skeleton of the plug-in itself. This illustrates incorporating a pspScript which includes a request for single-step/interactive execution mode, and thus the dialogues. It catches the terminate exception raised via the dialogue, and then terminates.
def generateWebImageSet(dasImage, dasLayer, title, mode):
try:
img = dasImage.duplicate()
# ...
bkg = img.layers[-1]
frameWidth = 52
start = bkg.offsets
end = (start[0]+bkg.width, start[1]+frameWidth)
# pspScript: (snippet included verbatim)
# SetExecutionMode / begin interactive single-step through pspScript
App.Do( Environment, 'SetExecutionMode', {
'GeneralSettings': {
'ExecutionMode': App.Constants.ExecutionMode.Interactive
}
})
# Selection
App.Do( Environment, 'Selection', {
'General' : {
'Mode' : 'Replace',
'Antialias' : False,
'Feather' : 0
},
'Start': start,
'End': end
})
# Promote
App.Do( Environment, 'SelectPromote' )
# und_so_weiter ...
except App.appTerminate:
raise AssertionError, "script cancelled"
# /generateWebImageSet
# _generateFloatingCanvasSetWeb.register -----------------------------------------
#
def generateFloatingCanvasSetWeb(dasImage, dasLayer, title):
mode="FCSW"
generateWebImageSet(dasImage, dasLayer, title, mode)
register(
"generateFloatingCanvasSetWeb",
"Generate Floating- Frame GW Canvas Image Set for Web Page",
"Generate Floating- Frame GW Canvas Image Set for Web Page",
"C G",
"C G",
"2019",
"<Image>/Image/Generate Web Imagesets/Floating-Frame Gallery-Wrapped Canvas Imageset...",
"*",
[
( PF_STRING, "title", "title", "")
],
[],
generateFloatingCanvasSetWeb)
main()
I realize that this may seem like a lot of work just to be able to include some pspScripts in a gimp plug-in, and to be able to single-step through the emulation. But we are talking about maybe 10K lines of scripts (and multiple scripts).
However, if any of this helps anyone else with dialogues inside plug-ins, etc., so much the better.
I want to use a function to read inputs file paths from a dataframe and send them to my snakemake rule. I also have a helper function to select the remote from which to pull the files.
from snakemake.remote.GS import RemoteProvider as GSRemoteProvider
from snakemake.remote.SFTP import RemoteProvider as SFTPRemoteProvider
from os.path import join
import pandas as pd
configfile: "config.yaml"
units = pd.read_csv(config["units"]).set_index(["library", "unit"], drop=False)
TMP= join('data', 'tmp')
def access_remote(local_path):
""" Connnects to remote as defined in config file"""
provider = config['provider']
if provider == 'GS':
GS = GSRemoteProvider()
remote_path = GS.remote(join("gs://" + config['bucket'], local_path))
elif provider == 'SFTP':
SFTP = SFTPRemoteProvider(
username=config['user'],
private_key=config['ssh_key']
)
remote_path = SFTP.remote(
config['host'] + ":22" + join(base_path, local_path)
)
else:
remote_path = local_path
return remote_path
def get_fastqs(wc):
"""
Get fastq files (units) of a particular library - sample
combination from the unit sheet.
"""
fqs = units.loc[
(units.library == wc.library) &
(units.libtype == wc.libtype),
"fq1"
]
return {
"r1": list(map(access_remote, fqs.fq1.values)),
}
# Combine all fastq files from the same sample / library type combination
rule combine_units:
input: unpack(get_fastqs)
output:
r1 = join(TMP, "reads", "{library}_{libtype}.end1.fq.gz")
threads: 12
run:
shell("cat {i1} > {o1}".format(i1=input['r1'], o1=output['r1']))
My config file contains the bucket name and provider, which are passed to the function. This works as expected when running simply snakemake.
However, I would like to use the kubernetes integration, which requires passing the provider and bucket name in the command line. But when I run:
snakemake -n --kubernetes --default-remote-provider GS --default-remote-prefix bucket-name
I get this error:
ERROR :: MissingInputException in line 19 of Snakefile:
Missing input files for rule combine_units:
bucket-name/['bucket-name/lib1-unit1.end1.fastq.gz', 'bucket-name/lib1-unit2.end1.fastq.gz', 'bucket-name/lib1-unit3.end1.fastq.gz']
The bucket is applied twice (once mapped correctly to each element, and once before the whole list (which gets converted to a string). Did I miss something ? Is there a good way to work around this ?
I met an error when running codes at the bottom. It's like a simple ftp.
I use python2.6.6 and CentOS release 6.8
In most linux server, it gets right results like this:(I'm very sorry that I have just sign up and couldn't )
Clinet:
[root#Test ftp]# python client.py
path:put|/home/aaa.txt
Server:
[root#Test ftp]# python server.py
connected...
pre_data:put|aaa.txt|4
cmd: put
file_name: aaa.txt
file_size: 4
upload successed.
But I get errors in some server(such as my own VM in my PC). I have done lots of tests(python2.6/python2.7, Centos6.5/Centos6.7) and found this error is not because them. Here is the error imformation:
[root#Lewis-VM ftp]# python server.py
connected...
pre_data:put|aaa.txt|7sdfsdf ###Here gets the wrong result, "sdfsdf" is the content of /home/aaa.txt and it shouldn't be sent here to 'file_size' and so it cause the "ValueError" below
cmd: put
file_name: aaa.txt
file_size: 7sdfsdf
----------------------------------------
Exception happened during processing of request from ('127.0.0.1', 10699)
Traceback (most recent call last):
File "/usr/lib64/python2.6/SocketServer.py", line 570, in process_request_thread
self.finish_request(request, client_address)
File "/usr/lib64/python2.6/SocketServer.py", line 332, in finish_request
self.RequestHandlerClass(request, client_address, self)
File "/usr/lib64/python2.6/SocketServer.py", line 627, in __init__
self.handle()
File "server.py", line 30, in handle
if int(file_size)>recv_size:
ValueError: invalid literal for int() with base 10: '7sdfsdf\n'
What's more, I found that if I insert a time.sleep(1) between sk.send(cmd+"|"+file_name+'|'+str(file_size)) and sk.send(data) in client.py, the error will disappear. I have said that I did tests in different system and python versions and the error is not because them. So I guess that is it because of some system configs? I have check about socket.send() and socket.recv() in python.org but fount nothing helpful. So could somebody help me to explain why this happend?
The code are here:
#!/usr/bin/env python
#coding:utf-8
################
#This is server#
################
import SocketServer
import os
class MyServer(SocketServer.BaseRequestHandler):
def handle(self):
base_path = '/home/ftp/file'
conn = self.request
print 'connected...'
while True:
#####receive pre_data: we should get data like 'put|/home/aaa|7'
pre_data = conn.recv(1024)
print 'pre_data:' + pre_data
cmd,file_name,file_size = pre_data.split('|')
print 'cmd: ' + cmd
print 'file_name: '+ file_name
print 'file_size: '+ file_size
recv_size = 0
file_dir = os.path.join(base_path,file_name)
f = file(file_dir,'wb')
Flag = True
####receive 1024bytes each time
while Flag:
if int(file_size)>recv_size:
data = conn.recv(1024)
recv_size+=len(data)
else:
recv_size = 0
Flag = False
continue
f.write(data)
print 'upload successed.'
f.close()
instance = SocketServer.ThreadingTCPServer(('127.0.0.1',9999),MyServer)
instance.serve_forever()
#!/usr/bin/env python
#coding:utf-8
################
#This is client#
################
import socket
import sys
import os
ip_port = ('127.0.0.1',9999)
sk = socket.socket()
sk.connect(ip_port)
while True:
input = raw_input('path:')
#####we should input like 'put|/home/aaa.txt'
cmd,path = input.split('|')
file_name = os.path.basename(path)
file_size=os.stat(path).st_size
sk.send(cmd+"|"+file_name+'|'+str(file_size))
send_size = 0
f= file(path,'rb')
Flag = True
#####read 1024 bytes and send it to server each time
while Flag:
if send_size + 1024 >file_size:
data = f.read(file_size-send_size)
Flag = False
else:
data = f.read(1024)
send_size+=1024
sk.send(data)
f.close()
sk.close()
The TCP is a stream of data. That is the problem. TCP do not need to keep message boundaries. So when a client calls something like
connection.send("0123456789")
connection.send("ABCDEFGHIJ")
then a naive server like
while True;
data = conn.recv(1024)
print data + "_"
may print any of:
0123456789_ABCDEFGHIJ_
0123456789ABCDEFGHIJ_
0_1_2_3_4_5_6_7_8_9_A_B_C_D_E_F_G_H_I_J_
The server has no chance to recognize how many sends client called because the TCP stack at client side just inserted data to a stream and the server must be able to process the data received in different number of buffers than the client used.
Your server must contain a logic to separate the header and the data. All of application protocols based on TCP use a mechanism to identify application level boundaries. For example HTTP separates headers and body by an empty line and it informs about the body length in a separate header.
Your program works correctly when server receives a header with the command, name and size in a separate buffer it it fails when client is fast enough and push the data into stream quickly and the server reads header and data in one chunk.
I'm trying to perform remote uploading and deploying via WLST.
I succeed to deploy applications already present in the path of the WebLogic server. But now, I would like to deploy one remotely available.
Is there any possibility to upload WAR/EAR on the WebLogic server via WLST before deploying it?
I'm running weblogic 12C.
I tried with:
java -cp wlserver/server/lib/weblogic.jar weblogic.Deployer -adminurl t3://WeblogicServer:7001 -user weblogic -password weblogic -deploy /tmp/HelloWorld.war -remote -upload
give me as return:
weblogic.Deployer invoked with options: -adminurl t3://WeblogicServer:7001 -user weblogic -deploy /tmp/HelloWorld.war -remote -upload
<6 juil. 2012 17 h 58 CEST> <Info> <J2EE Deployment SPI> <BEA-260121> <Initiating deploy operation for application, HelloWorld [archive: /tmp/HelloWorld.war], to configured targets.>
[J2EE Deployment SPI:260080]Unable to define the application install directory at "/tmp/helloworld.war". Directory could not be created or is a file.
And when i check on the server, nothing is deployed.
I found the solution:
We must not use the /tmp directory as source, because the weblogic.Deployer use this directory.
So the command is:
java -cp wlserver/server/lib/weblogic.jar weblogic.Deployer -adminurl
t3://WeblogicServer:7001 -user weblogic -password weblogic -deploy
/home/admin/HelloWorld.war -remote -upload
Yes, there is. Take a look at WLST deploy command reference (This link is for version 10gR3).
You can specify the remote and the upload options with true, since their default values are false.
PS 1: The remote option is not available on WebLogic 9.0 and older versions (but you can always get a newer weblogic.jar and try.
PS 2: You will need to have remote access to the administrative interfaces (usually ports 7001 and/or 7002). That means opening firewalls, etc.
PS 3: Please provide the version of your WebLogic server, as this might help us help you.
Written myself, can do undeploy + deploy. The deploy line is:
deploy( project, archive
, apptargets, 'stage', plan
, remote='true', upload='true', archiveVersion=archiveversion, planVersion=planversion )
disadvantage of using version number in deploy is that redeploy with plan is not working
Complete script: (line next code out yourself, I had to add some spaces to place it here)
from java.io import FileReader
from datetime import datetime
def deployApp( project, archive, archiveversion, plan, planversion, apptargets ):
#### UNDEPLOY app
try:
domainConfig()
cd('/')
found = false
for app in cmo.getAppDeployments():
name=app.getName()
try:
cd('/AppDeployments/' + name )
applicationName=get('ApplicationName')
if applicationName==project:
print '\n__ FOUND aplication: ', name
found = true
version=get('VersionIdentifier')
sourcePath=get('SourcePath')
planPath=get('PlanPath')
targets=''
for target in cmo.getTargets():
targetName=target.getName()
if targets=='':
targets=targetName
else:
targets=targets + ', ' + targetName
print ' applicationName:\t', applicationName
print ' version:\t\t', version
print ' sourcePath:\t', sourcePath
print ' planPath:\t', planPath
print ' targets:\t\t', targets
if version==archiveversion + '#' + planversion:
print '\nCorrect Application version found. No installation needed\n'
return
else:
print '\n__ REMOVING application: ',applicationName, ' version:', version
undeploy(project)
except:
print "IGNORE ERROR: (deployed applicaties gegevens ophalen) ", sys.exc_info()[0]
if found==false:
print "Application " + project + "not found."
except:
print "Failure in application remove part: ", sys.exc_info()[0]
raise
# get deploy targets
try:
if apptargets==None:
serverConfig()
cd('/')
targetList = []
for cluster in cmo.getClusters():
targetList.append( cluster.getName() )
apptargets = ",".join(targetList)
print 'USING default app target:', apptargets, " (apptargets not found in environmentfile)"
except:
print "Failure while creating default target", sys.exc_info()[0]
raise
#### DEPLOY app
try:
print '\n__ DEPLOYING application: ', project, archiveversion + '#' + planversion + ' target:' + apptargets
deploy( project, archive
, apptargets, 'stage', plan
, remote='true', upload='true', archiveVersion=archiveversion, planVersion=planversion )
except:
print "Failure in deployment part: ", sys.exc_info()[0]
raise
# connect to domein
# 1) in plain passsword
# 1) using encrypted domain credential in separate files
# 2) using encrypted domain credential in home directory user
def Connect( domainproperties ):
plainUser = domainproperties.getProperty("user")
plainPassword = domainproperties.getProperty("password")
if not (plainUser is None) and not (plainPassword is None):
print "Connecting with user & password"
connect( plainUser, plainPassword, domainproperties.getProperty("url"))
return
encryptedUserConfigFile = domainproperties.getProperty("encryptedCredentialsFile")
encryptionKeyFile = domainproperties.getProperty("encryptionKeyFile")
if not (encryptedUserConfigFile is None) and not (encryptionKeyFile is None):
print "Connecting with encryptedCredentialsFile: ", encryptedUserConfigFile, " and encryptionKeyFile: ", encryptionKeyFile
connect(userConfigFile=encryptedUserConfigFile, userKeyFile=encryptionKeyFile, url=domainproperties.getProperty("url"))
return
print "Connecting with encryptedCredentialsFile in home directory"
connect( url=domainproperties.getProperty("url") )
return
try:
print "____ running app_deploy.py at " + str(datetime.now())
project = sys.argv[1]
archive = sys.argv[3]
archiveversion = sys.argv[4]
plan = sys.argv[5]
planversion = sys.argv[6]
environmentfile = sys.argv[2]
propertyfilereader=FileReader(environmentfile)
domainproperties=Properties()
domainproperties.load(propertyfilereader)
print " environment: ", environmentfile
print " Ear: ", archive, " version:", archiveversion
print " plan: ", plan, " version:", planversion
## Connect
#connect( domainproperties.getProperty("user"), domainproperties.getProperty("password"), domainproperties.getProperty("url") )
Connect( domainproperties )
try:
## Deploy
deployApp( project, archive, archiveversion, plan, planversion, domainproperties.getProperty("apptargets") )
finally:
# niet dat het uitmaakt, maar wel zo netjes
disconnect()
except:
print "ABORT APP DEPLOY: ", # sys.exc_info()[0], "_", sys.exc_info()[1]
raise
Have fun with it.