Initial commit
This commit is contained in:
commit
351d98c523
36 changed files with 1836 additions and 0 deletions
0
pyjeeves/__init__.py
Normal file
0
pyjeeves/__init__.py
Normal file
26
pyjeeves/db.py
Normal file
26
pyjeeves/db.py
Normal file
|
|
@ -0,0 +1,26 @@
|
|||
# -*- coding: utf-8 -*-
|
||||
|
||||
from sqlalchemy import create_engine
|
||||
from sqlalchemy.orm.session import Session
|
||||
|
||||
from models.jvsmodels import Base
|
||||
|
||||
|
||||
class MySQLSession(Session):
|
||||
"""docstring for MySQLSession"""
|
||||
def __init__(self, settings):
|
||||
self.engine = create_engine(
|
||||
'mysql+pymysql://{user}:{passwd}@{host}:{port}/{db}'.format(**settings))
|
||||
super(MySQLSession, self).__init__(bind=self.engine)
|
||||
|
||||
def create_db(self):
|
||||
Base.metadata.create_all(self.engine)
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
import yaml
|
||||
with open("config.yml", 'r') as ymlfile:
|
||||
cfg = yaml.load(ymlfile)
|
||||
|
||||
session = MySQLSession(cfg['mysql'])
|
||||
session.create_db()
|
||||
151
pyjeeves/jvsquery.py
Normal file
151
pyjeeves/jvsquery.py
Normal file
|
|
@ -0,0 +1,151 @@
|
|||
# -*- coding: utf-8 -*-
|
||||
"""
|
||||
pyjeeves.jvsquery
|
||||
~~~~~~~~~~~~~~~~~~~~~~
|
||||
|
||||
Jeeves data queries
|
||||
"""
|
||||
import pymssql
|
||||
import datetime
|
||||
import logging
|
||||
|
||||
|
||||
class JvsQuery():
|
||||
"""docstring for JvsQuery"""
|
||||
def __init__(self, settings):
|
||||
super(JvsQuery, self).__init__()
|
||||
self.settings = settings
|
||||
self.logger = logging.getLogger("PyJeeves.jvsquery")
|
||||
|
||||
def _execute(self, query="", params=(), iterator=True):
|
||||
with pymssql.connect(**self.settings) as conn:
|
||||
with conn.cursor(as_dict=True) as cursor:
|
||||
cursor.execute(query, params)
|
||||
if iterator:
|
||||
for row in cursor:
|
||||
if cursor.rownumber % 1000 == 0 and cursor.rownumber != 0:
|
||||
self.logger.debug("Cursor is at pos %d" % cursor.rownumber)
|
||||
yield row
|
||||
else:
|
||||
return cursor.fetchall()
|
||||
|
||||
def _ft(self, updated_dt='2000-01-01 00:00:00.000',
|
||||
created_dt='2000-01-01 00:00:00.000', limit=None):
|
||||
query = (
|
||||
"""SELECT FaktNr, FaktRadnr, ForetagKod, FtgNr, OrderNr, OrdTyp, Saljare,
|
||||
KundKategoriKod, ArtNr, EnhetsKod, VaruGruppKod, Redovisnar, Period, FaktTB,
|
||||
FaktTG, FaktLevAnt, FaktLevAntAltEnh, FPris, FaktRadSumma, ValKod, ValKurs,
|
||||
RowCreatedDt, RowUpdatedDt, RowUpdatedBy, FaktDat
|
||||
FROM ft
|
||||
WHERE ft.ForetagKod = 1
|
||||
AND (ft.RowCreatedDt > %(created_dt)s
|
||||
OR ft.RowUpdatedDt > %(updated_dt)s)""")
|
||||
params = {'created_dt': created_dt, 'updated_dt': updated_dt}
|
||||
|
||||
return self._execute(query, params)
|
||||
|
||||
def _orp(self, updated_dt='2000-01-01 00:00:00.000',
|
||||
created_dt='2000-01-01 00:00:00.000', limit=None):
|
||||
query = (
|
||||
"""SELECT orp.OrderNr, OrdRadnr, OrdRadNrStrPos, orp.OrdRestNr, orp.ForetagKod,
|
||||
ArtNr, orp.FtgNr, vb_pris,
|
||||
OrdAntal, OrdAntalAltEnh, AltEnhetKod, OrdLevAntal, OrdLevAntalAltEnh,
|
||||
orp.FaktNr, orp.OrdDatum, orp.OrdBerLevDat, orp.OrdBerednDat, orp.OrdLevDat,
|
||||
orp.RowUpdatedBy, orp.RowUpdatedDt, orp.RowCreatedBy, orp.RowCreatedDt,
|
||||
salj.Saljare, salj.SaljareNamn,
|
||||
xs.OrdRadSt, xs.OrdRStatBeskr, x6.OrdTyp, x6.OrdTypBeskr
|
||||
FROM orp
|
||||
LEFT OUTER JOIN oh ON oh.OrderNr = orp.OrderNr
|
||||
AND oh.ForetagKod = orp.ForetagKod
|
||||
LEFT OUTER JOIN salj ON salj.Saljare = orp.Saljare
|
||||
AND salj.ForetagKod = orp.ForetagKod
|
||||
LEFT OUTER JOIN xs ON xs.OrdRadSt = orp.OrdRadSt
|
||||
AND xs.ForetagKod = orp.ForetagKod
|
||||
LEFT OUTER JOIN x6 ON x6.OrdTyp = orp.OrdTyp
|
||||
AND x6.ForetagKod = orp.ForetagKod
|
||||
WHERE orp.ForetagKod = 1
|
||||
AND (oh.RowCreatedDt > %(created_dt)s
|
||||
OR oh.RowUpdatedDt > %(updated_dt)s
|
||||
OR salj.RowCreatedDt > %(created_dt)s
|
||||
OR salj.RowUpdatedDt > %(updated_dt)s
|
||||
OR xs.RowCreatedDt > %(created_dt)s
|
||||
OR xs.RowUpdatedDt > %(updated_dt)s
|
||||
OR x6.RowCreatedDt > %(created_dt)s
|
||||
OR x6.RowUpdatedDt > %(updated_dt)s)""")
|
||||
params = {'created_dt': created_dt, 'updated_dt': updated_dt}
|
||||
|
||||
return self._execute(query, params)
|
||||
|
||||
def _ar(self, limit=None):
|
||||
query = (
|
||||
"""SELECT ArtNr, ar.VaruGruppKod, VaruGruppBeskr, ArtBeskr, ArtBeskr2, ArtBeskrSpec,
|
||||
ar.ArtProdKlass, ArtProdklBeskr, ar.ArtProdKonto, ArtProdKontoBeskr, ar.ArtKod,
|
||||
ArtTypBeskr, LagTyp, EnhetsKod, LevNr, ItemStatusCode, LagSaldoArtikel,
|
||||
ar.RowUpdatedBy, ar.RowUpdatedDt, ar.RowCreatedBy, ar.RowCreatedDt, ar.ForetagKod
|
||||
FROM ar
|
||||
LEFT OUTER JOIN arpk ON ar.ArtProdKonto = arpk.ArtprodKonto
|
||||
AND ar.ForetagKod = arpk.ForetagKod
|
||||
LEFT OUTER JOIN vg ON ar.VaruGruppKod = vg.VaruGruppKod
|
||||
AND ar.ForetagKod = vg.ForetagKod
|
||||
AND vg.SprakKod = 0
|
||||
LEFT OUTER JOIN xp ON ar.ArtProdKlass = xp.ArtProdKlass
|
||||
AND ar.ForetagKod = xp.ForetagKod
|
||||
LEFT OUTER JOIN xm ON ar.ArtKod = xm.ArtKod
|
||||
AND ar.ForetagKod = xm.ForetagKod
|
||||
WHERE ar.ForetagKod = 1""")
|
||||
|
||||
return self._execute(query)
|
||||
|
||||
def _kus(self, updated_dt='2000-01-01 00:00:00.000',
|
||||
created_dt='2000-01-01 00:00:00.000', limit=None):
|
||||
query = (
|
||||
"""SELECT kus.FtgNr, BetKod, kus.kundbetalarenr, kus.RowUpdatedBy, kus.RowUpdatedDt,
|
||||
kus.RowCreatedBy, kus.RowCreatedDt, kus.ForetagKod, kus.MakDateTime, kus.Makulerad,
|
||||
x1k.KundKategoriKod, x1k.KundKatBeskr,
|
||||
x1kk.Kundklass, x1kk.KundKlassBeskr,
|
||||
salj.Saljare, salj.SaljareNamn,
|
||||
OrgNr, FtgNamn, FtgPostAdr1, FtgPostAdr2, FtgPostAdr3,
|
||||
FtgPostadr4, FtgPostadr5, FtgPostnr, fr.LandsKod, FtgPostLevAdr3, FtgLevPostNr
|
||||
FROM kus
|
||||
LEFT OUTER JOIN fr ON fr.FtgNr = kus.FtgNr
|
||||
AND fr.ForetagKod = kus.ForetagKod
|
||||
LEFT OUTER JOIN x1k ON x1k.KundKategoriKod = kus.kundkategorikod
|
||||
AND x1k.ForetagKod = kus.ForetagKod
|
||||
LEFT OUTER JOIN x1kk ON x1kk.Kundklass = kus.kundklass
|
||||
AND x1kk.ForetagKod = kus.ForetagKod
|
||||
LEFT OUTER JOIN salj ON salj.Saljare = kus.Saljare
|
||||
AND salj.ForetagKod = kus.ForetagKod
|
||||
WHERE kus.ForetagKod = 1
|
||||
AND (kus.RowCreatedDt > %(created_dt)s
|
||||
OR kus.RowUpdatedDt > %(updated_dt)s
|
||||
OR fr.RowCreatedDt > %(created_dt)s
|
||||
OR fr.RowUpdatedDt > %(updated_dt)s
|
||||
OR x1k.RowCreatedDt > %(created_dt)s
|
||||
OR x1k.RowUpdatedDt > %(updated_dt)s
|
||||
OR x1kk.RowCreatedDt > %(created_dt)s
|
||||
OR x1kk.RowUpdatedDt > %(updated_dt)s
|
||||
OR salj.RowCreatedDt > %(created_dt)s
|
||||
OR salj.RowUpdatedDt > %(updated_dt)s)""")
|
||||
params = {'created_dt': created_dt, 'updated_dt': updated_dt}
|
||||
|
||||
return self._execute(query, params)
|
||||
|
||||
def get(self, jvs_tbl, updated_dt, created_dt):
|
||||
if not updated_dt:
|
||||
updated_dt = datetime.date(2000, 1, 1)
|
||||
if not created_dt:
|
||||
created_dt = datetime.date(2000, 1, 1)
|
||||
|
||||
updated_dt = updated_dt.strftime("%Y-%m-%d %H:%M:%S")
|
||||
created_dt = created_dt.strftime("%Y-%m-%d %H:%M:%S")
|
||||
|
||||
if jvs_tbl == 'Articles':
|
||||
return self._ar()
|
||||
elif jvs_tbl == 'Customers':
|
||||
return self._kus(updated_dt, created_dt)
|
||||
elif jvs_tbl == 'InvoiceRows':
|
||||
return self._ft(updated_dt, created_dt)
|
||||
elif jvs_tbl == 'OrderRows':
|
||||
return self._orp(updated_dt, created_dt)
|
||||
else:
|
||||
self.logger.warning("%s table has no get query" % jvs_tbl)
|
||||
71
pyjeeves/main.py
Normal file
71
pyjeeves/main.py
Normal file
|
|
@ -0,0 +1,71 @@
|
|||
# -*- coding: utf-8 -*-
|
||||
|
||||
import pprint
|
||||
import yaml
|
||||
import signal
|
||||
import sys
|
||||
|
||||
import logging
|
||||
import logging.config
|
||||
|
||||
from alembic.config import Config
|
||||
from alembic import command
|
||||
|
||||
from process import Process
|
||||
from jvsquery import JvsQuery
|
||||
from db import MySQLSession
|
||||
from utils import TaskThread
|
||||
|
||||
|
||||
pp = pprint.PrettyPrinter(indent=4)
|
||||
|
||||
|
||||
class SyncTread(TaskThread):
|
||||
"""docstring for ClassName"""
|
||||
def __init__(self, config):
|
||||
super(SyncTread, self).__init__()
|
||||
|
||||
jvs_query = JvsQuery(config['jeeves_db'])
|
||||
db_session = MySQLSession(config['mysql'])
|
||||
self.process = Process(jvs_query, db_session)
|
||||
|
||||
self.logger = logging.getLogger("PyJeeves.SyncTread")
|
||||
|
||||
def task(self):
|
||||
self.logger.info("Started sync")
|
||||
self.process.sync_data()
|
||||
self.logger.info("Finished sync")
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
with open("config.yml", 'r') as ymlfile:
|
||||
cfg = yaml.load(ymlfile)
|
||||
|
||||
logging.config.dictConfig(cfg['logging'])
|
||||
logger = logging.getLogger("PyJeeves")
|
||||
|
||||
logger.info("Running migrations")
|
||||
alembic_cfg = Config()
|
||||
for k in cfg['alembic']:
|
||||
alembic_cfg.set_main_option(k, cfg['alembic'][k])
|
||||
command.upgrade(alembic_cfg, "head")
|
||||
|
||||
logger.info("Application started")
|
||||
|
||||
def sigterm_handler(signal, frame):
|
||||
# save the state here or do whatever you want
|
||||
logger.info('Application interrupted')
|
||||
sys.exit(0)
|
||||
signal.signal(signal.SIGINT, sigterm_handler)
|
||||
signal.signal(signal.SIGTERM, sigterm_handler)
|
||||
|
||||
sync_thread = SyncTread(cfg)
|
||||
try:
|
||||
sync_thread.setInterval(cfg['sync_interval'])
|
||||
sync_thread.start()
|
||||
sync_thread.join()
|
||||
finally:
|
||||
sync_thread.shutdown()
|
||||
logger.info("Thread stopped")
|
||||
|
||||
logger.info("Application stopped")
|
||||
9
pyjeeves/models/__init__.py
Normal file
9
pyjeeves/models/__init__.py
Normal file
|
|
@ -0,0 +1,9 @@
|
|||
# -*- coding: utf-8 -*-
|
||||
"""
|
||||
pyjeeves.models
|
||||
~~~~~~~~~~~~~~~
|
||||
|
||||
consolodated models module
|
||||
"""
|
||||
|
||||
from .jvsmodels import * # noqa
|
||||
181
pyjeeves/models/jvsmodels.py
Normal file
181
pyjeeves/models/jvsmodels.py
Normal file
|
|
@ -0,0 +1,181 @@
|
|||
# -*- coding: utf-8 -*-
|
||||
"""
|
||||
pyjeeves.models
|
||||
~~~~~~~~~~~~~~~~~~~~~~
|
||||
|
||||
Jeeves data models
|
||||
"""
|
||||
|
||||
from sqlalchemy.ext.declarative import declarative_base
|
||||
from sqlalchemy import (Column, Integer, BigInteger, SmallInteger,
|
||||
String, ForeignKey, Numeric, DateTime, Boolean)
|
||||
from sqlalchemy.orm import relationship
|
||||
from sqlalchemy.schema import PrimaryKeyConstraint
|
||||
|
||||
Base = declarative_base()
|
||||
|
||||
|
||||
class InvoiceRows(Base):
|
||||
__tablename__ = 'jvs_invoice_rows'
|
||||
__table_args__ = (
|
||||
PrimaryKeyConstraint(name='invoice_rows_primary_key', mssql_clustered=True),)
|
||||
|
||||
FaktNr = Column(BigInteger, primary_key=True)
|
||||
FaktRadnr = Column(Integer, primary_key=True)
|
||||
ForetagKod = Column(SmallInteger, primary_key=True)
|
||||
|
||||
FtgNr = Column(String(length=16), ForeignKey(u'jvs_customers.FtgNr'))
|
||||
OrderNr = Column(BigInteger)
|
||||
OrdTyp = Column(SmallInteger)
|
||||
Saljare = Column(String(length=16))
|
||||
KundKategoriKod = Column(SmallInteger) # Can't have foreign key, as foreing data is mutalble
|
||||
ArtNr = Column(String(length=16)) # Can't have foreign key, as foreing data is mutalble
|
||||
EnhetsKod = Column(String(length=8))
|
||||
VaruGruppKod = Column(String(length=8))
|
||||
|
||||
Redovisnar = Column(SmallInteger)
|
||||
Period = Column(SmallInteger)
|
||||
FaktDat = Column(DateTime)
|
||||
FaktTB = Column(Numeric(precision=19, scale=4))
|
||||
FaktTG = Column(Numeric(precision=8, scale=3))
|
||||
FaktLevAnt = Column(Numeric(precision=15, scale=6))
|
||||
FaktLevAntAltEnh = Column(Numeric(precision=15, scale=6))
|
||||
FPris = Column(Numeric(precision=19, scale=4))
|
||||
FaktRadSumma = Column(Numeric(precision=19, scale=4))
|
||||
ValKod = Column(String(length=3))
|
||||
ValKurs = Column(Numeric(precision=22, scale=14))
|
||||
RowCreatedDt = Column(DateTime)
|
||||
RowUpdatedDt = Column(DateTime)
|
||||
RowUpdatedBy = Column(String(length=16))
|
||||
|
||||
customer = relationship(u'Customers')
|
||||
articles = relationship(u'Articles', foreign_keys=[ArtNr],
|
||||
primaryjoin='Articles.ArtNr == InvoiceRows.ArtNr')
|
||||
|
||||
|
||||
class OrderRows(Base):
|
||||
__tablename__ = 'jvs_order_rows'
|
||||
__table_args__ = (
|
||||
PrimaryKeyConstraint(name='order_rows_primary_key', mssql_clustered=True),)
|
||||
|
||||
OrderNr = Column(BigInteger, primary_key=True)
|
||||
OrdRadnr = Column(Integer, primary_key=True)
|
||||
OrdRadNrStrPos = Column(Integer, primary_key=True)
|
||||
OrdRestNr = Column(SmallInteger, primary_key=True)
|
||||
ForetagKod = Column(SmallInteger, primary_key=True)
|
||||
|
||||
ArtNr = Column(String(length=16))
|
||||
FtgNr = Column(String(length=16), ForeignKey(u'jvs_customers.FtgNr'))
|
||||
vb_pris = Column(Numeric(precision=19, scale=4))
|
||||
OrdAntal = Column(Numeric(precision=15, scale=6))
|
||||
OrdAntalAltEnh = Column(Numeric(precision=15, scale=6))
|
||||
AltEnhetKod = Column(String(length=16))
|
||||
OrdLevAntal = Column(Numeric(precision=15, scale=6))
|
||||
OrdLevAntalAltEnh = Column(Numeric(precision=15, scale=6))
|
||||
FaktNr = Column(BigInteger)
|
||||
OrdDatum = Column(DateTime)
|
||||
OrdBerLevDat = Column(DateTime)
|
||||
OrdBerednDat = Column(DateTime)
|
||||
OrdLevDat = Column(DateTime)
|
||||
|
||||
# Data from SalesPersonel (salj) in Jeeves
|
||||
Saljare = Column(String(length=32))
|
||||
SaljareNamn = Column(String(length=64))
|
||||
|
||||
# Data from OrderRowStatuses (xs) in Jeeves
|
||||
OrdRadSt = Column(SmallInteger)
|
||||
OrdRStatBeskr = Column(String(length=64))
|
||||
|
||||
# Data from OrderTypes (x6) in Jeeves
|
||||
OrdTyp = Column(SmallInteger)
|
||||
OrdTypBeskr = Column(String(length=64))
|
||||
|
||||
RowCreatedDt = Column(DateTime)
|
||||
RowCreatedBy = Column(String(length=16))
|
||||
RowUpdatedDt = Column(DateTime)
|
||||
RowUpdatedBy = Column(String(length=16))
|
||||
|
||||
customer = relationship(u'Customers')
|
||||
invoice_rows = relationship(u'InvoiceRows', foreign_keys=[FaktNr],
|
||||
primaryjoin='InvoiceRows.FaktNr == OrderRows.FaktNr')
|
||||
articles = relationship(u'Articles', foreign_keys=[ArtNr],
|
||||
primaryjoin='Articles.ArtNr == OrderRows.ArtNr')
|
||||
|
||||
|
||||
class Articles(Base):
|
||||
__tablename__ = 'jvs_articles'
|
||||
__table_args__ = (
|
||||
PrimaryKeyConstraint(name='articles_primary_key', mssql_clustered=True),)
|
||||
|
||||
ArtNr = Column(String(length=16), primary_key=True)
|
||||
ForetagKod = Column(SmallInteger, primary_key=True)
|
||||
|
||||
ArtBeskr = Column(String(length=128))
|
||||
ArtBeskr2 = Column(String(length=256))
|
||||
ArtBeskrSpec = Column(String(length=128))
|
||||
|
||||
ArtProdKonto = Column(String(length=8))
|
||||
ArtProdKontoBeskr = Column(String(length=64))
|
||||
VaruGruppKod = Column(String(length=8))
|
||||
VaruGruppBeskr = Column(String(length=64))
|
||||
ArtProdKlass = Column(String(length=8))
|
||||
ArtProdklBeskr = Column(String(length=64))
|
||||
ArtKod = Column(SmallInteger)
|
||||
ArtTypBeskr = Column(String(length=64))
|
||||
|
||||
LagTyp = Column(SmallInteger)
|
||||
EnhetsKod = Column(String(length=8))
|
||||
LevNr = Column(String(length=16))
|
||||
ItemStatusCode = Column(Integer)
|
||||
LagSaldoArtikel = Column(Numeric(precision=15, scale=6))
|
||||
|
||||
RowCreatedDt = Column(DateTime)
|
||||
RowCreatedBy = Column(String(length=16))
|
||||
RowUpdatedDt = Column(DateTime)
|
||||
RowUpdatedBy = Column(String(length=16))
|
||||
|
||||
|
||||
class Customers(Base):
|
||||
__tablename__ = 'jvs_customers'
|
||||
__table_args__ = (
|
||||
PrimaryKeyConstraint(name='customers_primary_key', mssql_clustered=True),)
|
||||
|
||||
FtgNr = Column(String(length=16), primary_key=True)
|
||||
ForetagKod = Column(SmallInteger, primary_key=True)
|
||||
|
||||
BetKod = Column(String(length=8))
|
||||
Saljare = Column(String(length=16))
|
||||
kundbetalarenr = Column(String(length=16))
|
||||
|
||||
Makulerad = Column(Boolean)
|
||||
MakDateTime = Column(DateTime)
|
||||
|
||||
RowCreatedDt = Column(DateTime)
|
||||
RowCreatedBy = Column(String(length=16))
|
||||
RowUpdatedDt = Column(DateTime)
|
||||
RowUpdatedBy = Column(String(length=16))
|
||||
|
||||
# Data from SalesPersonel (salj) in Jeeves
|
||||
Saljare = Column(String(length=32))
|
||||
SaljareNamn = Column(String(length=64))
|
||||
|
||||
# Data from CustomerCategories (x1k) in Jeeves
|
||||
KundKategoriKod = Column(SmallInteger)
|
||||
KundKatBeskr = Column(String(length=64))
|
||||
|
||||
# Data from CustomerClasses (x1kk) in Jeeves
|
||||
Kundklass = Column(String(length=16))
|
||||
KundKlassBeskr = Column(String(length=64))
|
||||
|
||||
# Data from Companies (fr) in Jeeves
|
||||
OrgNr = Column(String(length=32))
|
||||
FtgNamn = Column(String(length=64))
|
||||
FtgPostAdr1 = Column(String(length=64))
|
||||
FtgPostAdr2 = Column(String(length=64))
|
||||
FtgPostAdr3 = Column(String(length=64))
|
||||
FtgPostadr4 = Column(String(length=64))
|
||||
FtgPostadr5 = Column(String(length=128))
|
||||
FtgPostnr = Column(String(length=16))
|
||||
LandsKod = Column(String(length=16))
|
||||
FtgPostLevAdr3 = Column(String(length=64))
|
||||
FtgLevPostNr = Column(String(length=16))
|
||||
68
pyjeeves/process.py
Normal file
68
pyjeeves/process.py
Normal file
|
|
@ -0,0 +1,68 @@
|
|||
# -*- coding: utf-8 -*-
|
||||
"""
|
||||
pyjeeves.process
|
||||
~~~~~~~~~~~~~~~~~~~~~~
|
||||
|
||||
Jeeves data process
|
||||
"""
|
||||
|
||||
|
||||
from models import Articles, Customers, InvoiceRows, OrderRows
|
||||
|
||||
from sqlalchemy import desc
|
||||
from sqlalchemy.inspection import inspect
|
||||
|
||||
import logging
|
||||
|
||||
|
||||
class Process():
|
||||
"""docstring for Process"""
|
||||
def __init__(self, jvs_query, db_session):
|
||||
super(Process, self).__init__()
|
||||
self.query = jvs_query
|
||||
self.session = db_session
|
||||
self.logger = logging.getLogger("PyJeeves.process")
|
||||
|
||||
def _update_model(self, model, kwargs):
|
||||
for k, v in kwargs.items():
|
||||
if getattr(model, k) != v:
|
||||
if type(getattr(model, k)) is bool:
|
||||
setattr(model, k, bool(int(v)))
|
||||
else:
|
||||
setattr(model, k, v)
|
||||
|
||||
def _get_last_dates(self, model):
|
||||
_last_update = self.session.query(model).\
|
||||
order_by(desc(model.RowUpdatedDt)).first()
|
||||
_last_create = self.session.query(model).\
|
||||
order_by(desc(model.RowCreatedDt)).first()
|
||||
|
||||
return (_last_update.RowUpdatedDt if _last_update else None,
|
||||
_last_create.RowCreatedDt if _last_create else None)
|
||||
|
||||
def _sync_model(self, model, jvs_tbl='companies'):
|
||||
_p_keys = [key.name for key in inspect(model).primary_key]
|
||||
_last_update, _last_create = self._get_last_dates(model)
|
||||
_data = self.query.get(jvs_tbl, _last_update, _last_create)
|
||||
|
||||
if _data:
|
||||
self.logger.info("Syncing %s" % jvs_tbl)
|
||||
for item in _data:
|
||||
_filter_kwargs = {k: item.get(k) for k in _p_keys}
|
||||
_existing = self.session.query(model).\
|
||||
filter_by(**_filter_kwargs).first()
|
||||
|
||||
if _existing:
|
||||
self._update_model(_existing, item)
|
||||
else:
|
||||
_new = model(**item)
|
||||
self.session.add(_new)
|
||||
else:
|
||||
self.logger.info("No sync made for %s" % jvs_tbl)
|
||||
|
||||
def sync_data(self):
|
||||
self._sync_model(Customers, 'Customers')
|
||||
self._sync_model(Articles, 'Articles')
|
||||
self._sync_model(InvoiceRows, 'InvoiceRows')
|
||||
self._sync_model(OrderRows, 'OrderRows')
|
||||
self.session.commit()
|
||||
38
pyjeeves/utils.py
Normal file
38
pyjeeves/utils.py
Normal file
|
|
@ -0,0 +1,38 @@
|
|||
# -*- coding: utf-8 -*-
|
||||
"""
|
||||
pyjeeves.utils
|
||||
~~~~~~~~~~~~~~~~~~~~~~
|
||||
|
||||
Utils to improve usablity
|
||||
"""
|
||||
import threading
|
||||
|
||||
|
||||
class TaskThread(threading.Thread):
|
||||
"""Thread that executes a task every N seconds"""
|
||||
|
||||
def __init__(self):
|
||||
threading.Thread.__init__(self)
|
||||
self._finished = threading.Event()
|
||||
self._interval = 15.0
|
||||
|
||||
def setInterval(self, interval):
|
||||
"""Set the number of seconds we sleep between executing our task"""
|
||||
self._interval = interval
|
||||
|
||||
def shutdown(self):
|
||||
"""Stop this thread"""
|
||||
self._finished.set()
|
||||
|
||||
def run(self):
|
||||
while 1:
|
||||
if self._finished.isSet():
|
||||
return
|
||||
self.task()
|
||||
|
||||
# sleep for interval or until shutdown
|
||||
self._finished.wait(self._interval)
|
||||
|
||||
def task(self):
|
||||
"""The task done by this thread - override in subclasses"""
|
||||
pass
|
||||
Loading…
Add table
Add a link
Reference in a new issue