I have the following model:
class ServerSimpleConfigSerializer(mixins.GetCSConfigMixin, serializers.ModelSerializer):
mp_autoteambalance = serializers.BooleanField(label='Auto Team Balance', default=True, required=False)
mp_friendlyfire = serializers.BooleanField(label='Friendly Fire', default=False, required=False)
mp_autokick = serializers.BooleanField(label='Auto team-killer banning and idle client kicking', default=True, required=False)
# hostname = serializers.CharField(label='Hostname', max_length=75, required=True)
rcon_password = serializers.CharField(label='RCON Password', max_length=75, required=True)
sv_password = serializers.CharField(label='Server Password', max_length=75, required=False)
mp_startmoney = serializers.IntegerField(label='Start Money', required=False, validators=[MinValueValidator(800), MaxValueValidator(16000)])
mp_roundtime = serializers.FloatField(label='Round Time', required=False)
mp_timelimit = serializers.IntegerField(label='Map Time Limit', required=False)
fpath = os.path.join(settings.ROOT_DIR, "cs16/tmp/server.json")
class Meta:
model = CS16Server
fields = ('name', 'game_config', 'mp_autoteambalance', 'mp_friendlyfire',
'mp_autokick', 'hostname', 'rcon_password', 'sv_password',
'mp_startmoney', 'mp_roundtime', 'mp_timelimit')
read_only_fields = ('name', 'game_config',)
The model has the following fields:
name, game_config (big text) and hostname
How can I return the above defined fields for the serializer, although they are not present on the model ?
I would like to set some custom values for each field & return them as a JSON.
Is that possible ?
Actually the values for the above defined fields are found in "game_config" field.
I would like to parse those values & return them & I would not want to put them as separate fields in the model.
Parse game_config, obtain a pair of: (field0, val0) ... (fieldN, valN) and in the serializer,
set those values for the serializer fields.
For now I only get the following response:
{
"name": "Chronos",
"game_config": "hostname \"A New Gameservers.com Server is Born\"\nrcon_password \"\"\nsv_password \"1410271\"\nsv_contact email#domain.com\nsv_region 255\nsv_filterban 1\nsv_logbans 0\nsv_unlag 1\nmp_startmoney 800\nmp_chattime 30\nmp_footsteps 1\nsv_footsteps 1\nmp_logdetail 0\nmp_logmessages 0\nmp_timelimit 30\nmp_autokick 1\nmp_autoteambalance 1\nmp_flashlight 0\nmp_forcerespawn 0\nmp_forcechasecam 0\nmp_freezetime 0\nmp_friendlyfire 0\nmp_hostagepenalty 0\nmp_limitteams 0\nmp_roundtime 5\nmp_tkpunish 1\nsv_voiceenable 1\nsv_voicecodec voice_speex\nsv_voicequality 3\nsv_alltalk 0\nsv_restartround 1\nsv_maxspeed 320\nsv_proxies 1\nallow_spectators 1\nsv_allowupload 1\npausable 0\ndecalfrequency 40\nmp_falldamage 0\nsv_cheats 0\nsv_lan 0\nsv_maxrate 20000\nsv_minrate 4000\nexec listip.cfg",
"mp_autoteambalance": true,
"mp_friendlyfire": false,
"mp_autokick": true,
"hostname": "none"
}
Related
I'm creating the following connector:
CREATE
SOURCE CONNECTOR IF NOT EXISTS `myconn` WITH (
"name" = 'myconn',
"connector.class" = 'MySqlCdcSource',
"tasks.max" = 1,
--Database config--------------------------
"database.hostname" = '${dbHost}',
"database.port"....
--Kafka config------------------------------
"kafka.api.key" = '${kafkaApiKey}',
"kafka.auth.mode"...
--Connector behavior------------------------
"output.data.format" = 'AVRO',
"output.key.format" = 'STRING',
"key.converter" = 'org.apache.kafka.connect.storage.StringConverter',
"key.converter.schemas.enable" = true,
"tombstones.on.delete" = true,
"null.handling.mode" = 'keep',
"include.schema.changes" = false,
"table.include.list" = 'sandbox\.(xxx|yyy)',
"errors.tolerance" = 'none',
"errors.log.enable" = true,
"errors.log.include.messages" = true,
--Topics configuration----------------------
"topic.creation.default.cleanup.policy" = 'compact',
"topic.creation.default.min.insync.replicas"...
--Predicates--------------------------------
"predicates" = 'TopicDoestHaveIdField',
"predicates.TopicDoestHaveIdField.type" = 'org.apache.kafka.connect.transforms.predicates.TopicNameMatches',
"predicates.TopicDoestHaveIdField.pattern" = 'myconn.sandbox\.(zzz|ooo)$',
--Transforms--------------------------------
"transforms" = 'extractKey',
"transforms.extractKey.type" = 'org.apache.kafka.connect.transforms.ExtractField$Key',
"transforms.extractKey.field" = 'id',
"transforms.extractKey.predicate" = 'TopicDoestHaveIdField',
"transforms.extractKey.negate" = true
);
In local development I'm working with io.debezium.connector.mysql.MySqlConnector and the transformation works correctly, the problem is when I make de creation in Confluent Cloud (using MySqlCdcSource). It gives me the following error:
The field configured for org.apache.kafka.connect.transforms.ExtractField transform does not exist in the key or value of Kafka records. Please verify the record's key or value has the configured field.
The problem is that it doesn't find the field Id, but when I make a test without the transformation, I see this key in the topic: Struct(id=0000), so the field exists. I suppose it should be related with types or some configuration option is missed in my connector. Any ideas?
The problem was related with a wrong usage of a predicate. Take a look at the following part:
--Predicates--------------------------------
"predicates" = 'TopicDoestHaveIdField',
"predicates.TopicDoestHaveIdField.type" = 'org.apache.kafka.connect.transforms.predicates.TopicNameMatches',
"predicates.TopicDoestHaveIdField.pattern" = 'myconn.sandbox\.(zzz|ooo)$',
--Transforms--------------------------------
"transforms" = 'extractKey',
"transforms.extractKey.type" = 'org.apache.kafka.connect.transforms.ExtractField$Key',
"transforms.extractKey.field" = 'id',
"transforms.extractKey.predicate" = 'TopicDoestHaveIdField',
"transforms.extractKey.negate" = true
As you can see negate is being used, so extractKey transforms includes all the tables that doesn't match with it. Despite that I was using "table.include.list", some topics where included additionally and they didn't have id field.
In my case the confusion was motivated because of the message in log:
The field configured for org.apache.kafka.connect.transforms.ExtractField transform does not exist in the key or value of Kafka records. Please verify the record's key or value has the configured field.
(No information about the involved topic) I was supposing that the message was talking about any of the both allowed tables ('zzz' or 'ooo') but in the end it was another third topic included.
I have created a minimal REST API using Flask, SQLAlchemy, and Marshmallow. here is app.py file:
from flask import Flask, request, jsonify
from flask_sqlalchemy import SQLAlchemy
from flask_marshmallow import Marshmallow
import os
# Initialize App
app = Flask(__name__)
basedir = os.path.abspath(os.path.dirname(__file__))
# Database Setup
app.config['SQLALCHEMY_DATABASE_URI'] = 'sqlite:///' + os.path.join(basedir, 'db.sqlite')
app.config['SQLALCHEMY_TRACK_MODIFICATIONS'] = False
# Init db
db = SQLAlchemy(app)
# Init marshmallow
ma = Marshmallow(app)
# Product Class/Model
class Product(db.Model):
id = db.Column(db.Integer, primary_key=True)
name = db.Column(db.String(100), unique=True)
description = db.Column(db.String(200))
price = db.Column(db.Float)
qty = db.Column(db.Integer)
def __init__(self, name, description, price, qty):
self.name = name
self.description = description
self.price = price
self.qty = qty
# Product Schema
class ProductSchema(ma.Schema):
class Meta:
fields = ('id', 'name', 'description', 'price', 'qty')
# Init Schema
product_schema = ProductSchema()
products_schema = ProductSchema(many=True)
# Create Product
#app.route('/product', methods=['POST'])
def add_product():
name = request.json['name']
description = request.json['description']
price = request.json['price']
qty = request.json['qty']
new_product = Product(name, description, price, qty)
db.session.add(new_product)
db.session.commit()
return product_schema.jsonify(new_product)
# Get All Products
#app.route('/receive', methods=['GET'])
def get_products():
all_products = Product.query.all()
result = products_schema.dump(all_products)
return jsonify(result)
# Run the Server
if __name__ == '__main__':
app.run(debug=True)
I want to extract the sender's IP address through the GET method. However, the sender's IP doesn't need to be part of the JSON payload.
Example: POST
{
"name": "Product 1",
"description": "This is product 1",
"price": 120.00,
"qty": 100
}
GET
{
"ip": "<whatever-the-ip>"
"name": "Product 1",
"description": "This is product 1",
"price": 120.00,
"qty": 100
}
How do I implement this functionality in my code? I tried using request.remote_addr, but I am not getting what I expected.
You can modify the response in the GET route any way you want before returning it, I added the request's IP to the response after serializing the db result like this:
# Get All Products
#app.route('/receive', methods=['GET'])
def get_products():
all_products = Product.query.all()
results = products_schema.dump(all_products)
for product in results
product.update({"ip": str(request.remote_addr)})
return jsonify(results)
The description for replaceField SMT says it can Filter or rename fields within a Struct or Map. However I can't find any working example for replacing or renaming fields within a struct.
I've got data in a topic being written into ElasticSearch using Kafka Connect Elasticsearch Sink. For simplicity, assume the format of the data looks like this.
{
'ID':22,
'ITEM': 'Shampoo'
'USER':{
'NAME': 'jon',
'AGE':25
}
}
So if I'm trying to rename/replace USER.NAME or USER.AGE, how would I configure that in the connector? (I've written everything in ksqldb). This is my current config where I rename ITEM to product and ID to id
CREATE SINK CONNECTOR ELASTIC_SINK WITH (
'connector.class' = 'io.confluent.connect.elasticsearch.ElasticsearchSinkConnector',
'connection.url' = 'http://host.docker.internal:9200',
'type.name' = '_doc',
'topics' = 'ELASTIC_TOPIC',
'key.ignore' = 'false',
'schema.ignore' = 'true',
'transforms' = 'RenameField',
'transforms.RenameField.type' = 'org.apache.kafka.connect.transforms.ReplaceField$Value',
'transforms.RenameField.renames' = 'ITEM:product,ID:id',
);
Take a look at the existing SO question and answer: https://stackoverflow.com/a/56601093/4778022
You can provide the path to the field to rename, with parts separated by periods.
CREATE SINK CONNECTOR ELASTIC_SINK WITH (
'connector.class' = 'io.confluent.connect.elasticsearch.ElasticsearchSinkConnector',
'connection.url' = 'http://host.docker.internal:9200',
'type.name' = '_doc',
'topics' = 'ELASTIC_TOPIC',
'key.ignore' = 'false',
'schema.ignore' = 'true',
'transforms' = 'RenameField',
'transforms.RenameField.type' = 'org.apache.kafka.connect.transforms.ReplaceField$Value',
'transforms.RenameField.renames' = 'USER.NAME:name,ITEM:product,ID:id',
);
first-time poster. I’m trying to create a simple contact option using Django Rest Framework. The contact page would allow users and non-users to send the site admin an email via a form. Been at this for weeks… I’ve added my questions and code below.
1) Is it the viewset that needs some additional work to connect to the form data?
2) Does the DRF API viewer allow for testing this out? Should it be showing the email fields?
# serializers.py
class CommentSerializer(serializers.Serializer):
email = serializers.EmailField()
message = serializers.CharField()
name = serializers.CharField()
# views.py
class CommentViewSet(viewsets.ViewSet):
def list(self, request): #, format=None
comment = CommentSerializer(data=request.data)
if comment.is_valid():
form_email = comment.data['email']
form_message = comment.data['message'] + "email: " + form_email
form_name = comment.data['name']
send_mail("New contact form submission",
form_message,
form_email,
['myemailaddress#gmail.com'],
fail_silently=False
)
return Response(comment.data)
# Not sure how the html connects here:
# return render('comment.html', {
# 'form': form_class,
# })
return Response(
{
"success": False,
'error-code':'invalid-data'
},
)
# urls.py
router = DefaultRouter()
router.register(r'profiles', views.ProfileViewSet)
router.register(r'users', views.UserViewSet)
router.register(r'comment', views.CommentViewSet, 'Comment')
urlpatterns = [
url(r'^', include(router.urls)),
]
I try to use the django REST-framework's tutorial http://django-rest-framework.org/#django-rest-framework to administrate users. (I also use the Neo4j database and the neo4django mapper https://github.com/scholrly/neo4django to acces data via python.)
Whatever, wen I call localhost:8000/users an AttributeError appears.
models.py
from django.utils import timezone
from django.conf import settings
from django.contrib.auth import models as django_auth_models
from ..db import models
from ..db.models.manager import NodeModelManager
from ..decorators import borrows_methods
class UserManager(NodeModelManager, django_auth_models.UserManager):
pass
# all non-overriden methods of DjangoUser are called this way instead.
# inheritance would be preferred, but isn't an option because of conflicting
# metaclasses and weird class side-effects
USER_PASSTHROUGH_METHODS = (
"__unicode__", "natural_key", "get_absolute_url",
"is_anonymous", "is_authenticated", "get_full_name", "set_password",
"check_password", "set_unusable_password", "has_usable_password",
"get_group_permissions", "get_all_permissions", "has_perm", "has_perms",
"has_module_perms", "email_user", 'get_profile','get_username')
#borrows_methods(django_auth_models.User, USER_PASSTHROUGH_METHODS)
class User(models.NodeModel):
objects = UserManager()
username = models.StringProperty(indexed=True, unique=True)
first_name = models.StringProperty()
last_name = models.StringProperty()
email = models.EmailProperty(indexed=True)
password = models.StringProperty()
is_staff = models.BooleanProperty(default=False)
is_active = models.BooleanProperty(default=False)
is_superuser = models.BooleanProperty(default=False)
last_login = models.DateTimeProperty(default=timezone.now())
date_joined = models.DateTimeProperty(default=timezone.now())
USERNAME_FIELD = 'username'
REQUIRED_FIELDS=['email']
serializers.py
from neo4django.graph_auth.models import User
from rest_framework import serializers
class UserSerializer(serializers.HyperlinkedModelSerializer):
class Meta:
model = User
fields = ('url', 'username', 'email')
views.py
from neo4django.graph_auth.models import User
from rest_framework import viewsets
from api.serializers import UserSerializer
class UserViewSet(viewsets.ModelViewSet):
"""
API endpoint that allows users to be viewed or edited.
"""
queryset = User.objects.all()
serializer_class = UserSerializer
I get this:
Environment:
Request Method: GET
Request URL: http://localhost:8000/users/
Django Version: 1.5.3
Python Version: 2.7.3
Installed Applications:
('core.models',
'django.contrib.auth',
'django.contrib.contenttypes',
'django.contrib.sessions',
'neo4django.graph_auth',
'rest_framework')
Installed Middleware:
('django.middleware.common.CommonMiddleware',
'django.contrib.sessions.middleware.SessionMiddleware',
'django.middleware.csrf.CsrfViewMiddleware',
'django.contrib.auth.middleware.AuthenticationMiddleware',
'django.contrib.messages.middleware.MessageMiddleware')
Traceback:
File "/opt/phaidra/env/local/lib/python2.7/site-packages/django/core/handlers/base.py" in get_response
115. response = callback(request, *callback_args, **callback_kwargs)
File "/opt/phaidra/env/local/lib/python2.7/site-packages/rest_framework/viewsets.py" in view
78. return self.dispatch(request, *args, **kwargs)
File "/opt/phaidra/env/local/lib/python2.7/site-packages/django/views/decorators/csrf.py" in wrapped_view
77. return view_func(*args, **kwargs)
File "/opt/phaidra/env/local/lib/python2.7/site-packages/rest_framework/views.py" in dispatch
399. response = self.handle_exception(exc)
File "/opt/phaidra/env/local/lib/python2.7/site-packages/rest_framework/views.py" in dispatch
396. response = handler(request, *args, **kwargs)
File "/opt/phaidra/env/local/lib/python2.7/site-packages/rest_framework/mixins.py" in list
92. serializer = self.get_pagination_serializer(page)
File "/opt/phaidra/env/local/lib/python2.7/site-packages/rest_framework/generics.py" in get_pagination_serializer
113. return pagination_serializer_class(instance=page, context=context)
File "/opt/phaidra/env/local/lib/python2.7/site-packages/rest_framework/pagination.py" in __init__
85. self.fields[results_field] = object_serializer(source='object_list', **context_kwarg)
File "/opt/phaidra/env/local/lib/python2.7/site-packages/rest_framework/serializers.py" in __init__
162. self.fields = self.get_fields()
File "/opt/phaidra/env/local/lib/python2.7/site-packages/rest_framework/serializers.py" in get_fields
198. default_fields = self.get_default_fields()
File "/opt/phaidra/env/local/lib/python2.7/site-packages/rest_framework/serializers.py" in get_default_fields
599. while pk_field.rel and pk_field.rel.parent_link:
Exception Type: AttributeError at /users/
Exception Value: 'IdLookup' object has no attribute 'rel'
I am rather new on the Python/Django/REST service area. I hope someone could help. Thanks in advance.
Firstly I would recommend to either use Tastypie - which is out of the box supported by neo4django - instead of Django Rest Framework - or use Django Rest Framework Serializer instead of the ModelSerializer or (worst choice in my opinion) HyperlinkedModelSerializer.
As for the error you get, the problem is that neo4django does not return the record id, therefore you get the error.
One solution is to override the restore_object function, like this, to include the ID.
# User Serializer
class UserSerializer(serializers.Serializer):
id = serializers.IntegerField()
username = serializers.CharField(max_length=30)
first_name = serializers.CharField(max_length=30)
last_name = serializers.CharField(max_length=30)
email = serializers.EmailField()
password = serializers.CharField(max_length=128)
is_staff = serializers.BooleanField()
is_active = serializers.BooleanField()
is_superuser = serializers.BooleanField()
last_login = serializers.DateTimeField()
date_joined = serializers.DateTimeField()
def restore_object(self, attrs, instance=None):
"""
Given a dictionary of deserialized field values, either update
an existing model instance, or create a new model instance.
"""
if instance is not None:
instance.id = attrs.get('ID', instance.pk)
instance.username = attrs.get('Username', instance.username)
instance.first_name = attrs.get('First Name', instance.first_name)
instance.last_name = attrs.get('First Name', instance.last_name)
instance.email = attrs.get('email', instance.email)
instance.password = attrs.get('Password', instance.password)
instance.is_staff = attrs.get('Staff', instance.is_staff)
instance.is_active = attrs.get('Active', instance.is_active)
instance.is_superuser = attrs.get('Superusers', instance.is_superuser)
instance.last_login = attrs.get('Last Seen', instance.last_login)
instance.date_joined = attrs.get('Joined', instance.date_joined)
return instance
return User(**attrs)
But I still think it's better to use Tastypie with ModelResource.
Here's a gist by Matt Luongo (or Lukas Martini ?) https://gist.github.com/mhluongo/5789513
TIP. Where Serializer in Django Rest Framework, is Resource in Tastypie and always make sure you are using the github version of neo4django (pip install -e git+https://github.com/scholrly/neo4django/#egg=neo4django)