python Archives | Eli's Blog

Capturing Solaris 11.4 (12) Analytics By Using Remote Administration Daemon (RAD) – Part 5

February 8, 2018February 8, 2018 Eli Kleinman Leave a comment

(1 votes, average: 5.00 out of 5)

Capturing Your Own Solaris 11.4 (12) Analytics By Using Remote Administration Daemon (RAD) – Part 5 Note: The full code is available in a GitHub repository. to play with the code, feel free to clone the Solaris Analytics Publisher Repository. This is part 5 out of a series of articles on how to use Oracle […]

Creating, Hosting Your Own CoreOS Rkt App Container Images(ACI)

January 2, 2018January 11, 2018 Eli Kleinman Leave a comment

(1 votes, average: 5.00 out of 5)

Creating, Hosting your own CoreOS Rkt (ACI) Images Creating your own (ACI) package The document below uses the ACI image format since we are dealing with Rkt. in the future Rkt might switch to the OCI format. to note Docker is using the OCI image format, which is part of the opencontainer initiative. The steps […]

Kubernetes CoreOS Ignition Configuration Generator – Part 6

December 5, 2017December 26, 2018 Eli Kleinman 2 Comments

(No Ratings Yet)

Kubernetes CoreOS Ignition Configuration Generator Written in Python. In the previous post I went through how to Configuring Kubernetes Træfik Ingress Controller, DNS, Dashboard. below are examples to use a small Python script I have written to automate most of the Kubernetes deployment process. Note: Get going in minutes with a full Kubernetes cluster by […]

Python remote backup script

September 13, 2016April 27, 2017 Eli Kleinman Leave a comment

(No Ratings Yet)

#!/bin/python

import os
import sys
import time
import datetime
import subprocess
import smtplib
from email.mime.text import MIMEText

def mysqlBackup():

    mysql_host = 'localhost'
    mysql_user = 'root'
    mysql_user_passwd = 'password'
    backup_path = '/mystuff/mysql_backup/'
    dblist = ['wordpress']
    datetime = time.strftime('%m%d%Y-%H%M%S')

    for db in dblist:
       backupdate = backup_path + db + "_" + datetime
       logger("creating backup folder - " + backupdate)
       if not os.path.exists(backupdate):
          os.makedirs(backupdate)

       xmldumpcmd = "/bin/mysqldump -u " + mysql_user + " -p" + mysql_user_passwd + " " + db + " --xml" + " > " + backupdate + "/" + db + ".xml"
       os.system(xmldumpcmd)
       logger(db + " XML backup complete, as... " + db + ".xml")
       sqldumpcmd = "/bin/mysqldump -u " + mysql_user + " -p" + mysql_user_passwd + " " + db + " > " + backupdate + "/" + db + ".sql"
       os.system(sqldumpcmd)
       logger(db + " SQL backup complete, as... " + db + ".sql")

    logger("Backup script completed")
    logger("Backups has been created in '" + backupdate + "' directory")

def syncRemote():

    logger("****************** rsync starting " + time.strftime("%m-%d-%Y %H:%M:%S") + "***********************")
    dir_list = ['/mystuff', '/var/tmp']

    for i in dir_list:
        logger("****************** Now syncing: " + i + " ******************")
        port = "'ssh -p 60022 -i /mystuff/backup_scripts/rsync-key'"
        #key = ' -i /mystuff/backup_scripts/rsync-key'
        cmd = '/bin/rsync -auvz -e ' + port + ' --progress ' + i + ' elik-remote@eli102.asuscomm.com:backup/'
        logger(cmd)

        progress = subprocess.check_output(cmd, shell=True)
        #progress = subprocess.Popen(cmd, shell=True, stdin=subprocess.PIPE, stdout=subprocess.PIPE)
        #for line in iter(progress.stdout.readline, b''):
            #logger(line.rstrip())
        logger(progress)

    logger("****************** rsync completed " + time.strftime("%m-%d-%Y %H:%M:%S") + " ***********************")

def logger(log):
    f.write(log + "\n")
    sys.stdout.flush()

def mailer():

    backupfile = '/var/tmp/backup.log'
    me = "admin@domain.com"
    you = "admin@domain.com"
    if (os.path.isfile(backupfile)):
       fp = open(backupfile, 'rb')

       msg = MIMEText(fp.read())
       fp.close()

       msg['Subject'] = 'Backup status - output %s' % backupfile
       msg['From'] = me 
       msg['To'] = you
       s = smtplib.SMTP('localhost')
       s.sendmail(me, [you], msg.as_string())
       s.quit()
       os.rename(backupfile, backupfile+"-old")

    else:
       msg = MIMEText("There was an error completing the backup")
       msg['Subject'] = 'Backup error opening file - %s' % backupfile
       msg['Subject'] = 'Backup error '
       s = smtplib.SMTP('localhost')
       s.sendmail(me, [you], msg.as_string())
       s.quit()

if __name__ == '__main__':
    f = open('/var/tmp/backup.log','a+')
    f.write("+++++++++++++++++++++ START TIME ** " + time.strftime("%m-%d-%Y %H:%M:%S") + " +++++++++++++++++++++\n")
    sys.stdout.flush()
    mysqlBackup()
    syncRemote()
    f.write("+++++++++++++++++++++ END TIME ** " + time.strftime("%m-%d-%Y %H:%M:%S") + " +++++++++++++++++++++\n")
    f.close()
    mailer()

#!/bin/python

import os

import sys

import time

import datetime

import subprocess

import smtplib

from email.mime.text import MIMEText

def mysqlBackup():

mysql_host = 'localhost'

mysql_user = 'root'

mysql_user_passwd = 'password'

backup_path = '/mystuff/mysql_backup/'

dblist = ['wordpress']

datetime = time.strftime('%m%d%Y-%H%M%S')

for db in dblist:

backupdate = backup_path + db + "_" + datetime

logger("creating backup folder - " + backupdate)

if not os.path.exists(backupdate):

os.makedirs(backupdate)

xmldumpcmd = "/bin/mysqldump -u " + mysql_user + " -p" + mysql_user_passwd + " " + db + " --xml" + " > " + backupdate + "/" + db + ".xml"

os.system(xmldumpcmd)

logger(db + " XML backup complete, as... " + db + ".xml")

sqldumpcmd = "/bin/mysqldump -u " + mysql_user + " -p" + mysql_user_passwd + " " + db + " > " + backupdate + "/" + db + ".sql"

os.system(sqldumpcmd)

logger(db + " SQL backup complete, as... " + db + ".sql")

logger("Backup script completed")

logger("Backups has been created in '" + backupdate + "' directory")

def syncRemote():

logger("****************** rsync starting " + time.strftime("%m-%d-%Y %H:%M:%S") + "***********************")

dir_list = ['/mystuff', '/var/tmp']

for i in dir_list:

logger("****************** Now syncing: " + i + " ******************")

port = "'ssh -p 60022 -i /mystuff/backup_scripts/rsync-key'"

#key = ' -i /mystuff/backup_scripts/rsync-key'

cmd = '/bin/rsync -auvz -e ' + port + ' --progress ' + i + ' elik-remote@eli102.asuscomm.com:backup/'

logger(cmd)

progress = subprocess.check_output(cmd, shell=True)

#progress = subprocess.Popen(cmd, shell=True, stdin=subprocess.PIPE, stdout=subprocess.PIPE)

#for line in iter(progress.stdout.readline, b''):

#logger(line.rstrip())

logger(progress)

logger("****************** rsync completed " + time.strftime("%m-%d-%Y %H:%M:%S") + " ***********************")

def logger(log):

f.write(log + "\n")

sys.stdout.flush()

def mailer():

backupfile = '/var/tmp/backup.log'

me = "admin@domain.com"

you = "admin@domain.com"

if (os.path.isfile(backupfile)):

fp = open(backupfile, 'rb')

msg = MIMEText(fp.read())

fp.close()

msg['Subject'] = 'Backup status - output %s' % backupfile

msg['From'] = me

msg['To'] = you

s = smtplib.SMTP('localhost')

s.sendmail(me, [you], msg.as_string())

s.quit()

os.rename(backupfile, backupfile+"-old")

else:

msg = MIMEText("There was an error completing the backup")

msg['Subject'] = 'Backup error opening file - %s' % backupfile

msg['Subject'] = 'Backup error '

s = smtplib.SMTP('localhost')

s.sendmail(me, [you], msg.as_string())

s.quit()

if __name__ == '__main__':

f = open('/var/tmp/backup.log','a+')

f.write("+++++++++++++++++++++ START TIME ** " + time.strftime("%m-%d-%Y %H:%M:%S") + " +++++++++++++++++++++\n")

sys.stdout.flush()

mysqlBackup()

syncRemote()

f.write("+++++++++++++++++++++ END TIME ** " + time.strftime("%m-%d-%Y %H:%M:%S") + " +++++++++++++++++++++\n")

f.close()

mailer()

Example Of Capturing IOStat With Python

September 7, 2016July 18, 2017 Eli Kleinman 2 Comments

(No Ratings Yet)

The below Example captures IOStat output with Python

#!/bin/python

import subprocess
import csv

io_c = subprocess.Popen(['iostat', '-xnsMr', '1', '2'], stdout=subprocess.PIPE, shell=False, stderr=subprocess.PIPE)
stdout = io_c.communicate()[0]
lc = len(stdout.split('\n'))
it_lc = lc / 2 + 2
reader = csv.reader(stdout.split('\n')[it_lc:], delimiter=',')
for row in reader:
    if any(row):
       #print row
       print row[0],row[1],row[2],row[3],row[7],row[8],row[9],row[10]

#!/bin/python

import subprocess

import csv

io_c = subprocess.Popen(['iostat', '-xnsMr', '1', '2'], stdout=subprocess.PIPE, shell=False, stderr=subprocess.PIPE)

stdout = io_c.communicate()[0]

lc = len(stdout.split('\n'))

it_lc = lc / 2 + 2

reader = csv.reader(stdout.split('\n')[it_lc:], delimiter=',')

for row in reader:

if any(row):

#print row

print row[0],row[1],row[2],row[3],row[7],row[8],row[9],row[10]

Output would look like

0.0 0.0 0.0 0.0 0.0 0 0 c3t3d0
0.0 0.0 0.0 0.0 0.0 0 0 c3t0d0
0.0 0.0 0.0 0.0 0.0 0 0 c3t1d0
0.0 0.0 0.0 0.0 0.0 0 0 c3t2d0
0.0 0.0 0.0 0.0 0.0 0 0 nas-srv:/export/sys-admin/unix
0.0 0.0 0.0 0.0 0.0 0 0 10.10.10.11:/export/shares

0.0 0.0 0.0 0.0 0.0 0 0 c3t3d0

0.0 0.0 0.0 0.0 0.0 0 0 c3t0d0

0.0 0.0 0.0 0.0 0.0 0 0 c3t1d0

0.0 0.0 0.0 0.0 0.0 0 0 c3t2d0

0.0 0.0 0.0 0.0 0.0 0 0 nas-srv:/export/sys-admin/unix

0.0 0.0 0.0 0.0 0.0 0 0 10.10.10.11:/export/shares

Capturing Your Own Solaris 11.4 (12) Analytics / Sstore – Part 3

August 31, 2016February 8, 2018 Eli Kleinman Leave a comment

(No Ratings Yet)

Capturing Your Own Solaris 11.4 (12) Analytics / Sstore Note: There is an updated process for Solaris 11.4+, please check out part 5 – Enhanced method on how to capture analytics in Solaris 11.4+ Note: The full code is available in a GitHub repository. to play with the code, feel free to just clone the […]

Capturing Your Own Solaris 11.4 (12) Analytics / Sstore – Part 2

August 31, 2016February 8, 2018 Eli Kleinman Leave a comment

(No Ratings Yet)

Capturing Solaris 11.4/(12) analytical data Note: There is an updated process for Solaris 11.4+, please check out part 5 – Enhanced method on how to capture analytics in Solaris 11.4+ Note: The full code is available in a GitHub repository. to play with the code, feel free to just clone the Solaris Analytics Publisher repository. […]

How To Create a Python Socket Server

August 24, 2016March 27, 2017 Eli Kleinman Leave a comment

(No Ratings Yet)

Create the Python Socket Server

#!/bin/python

import socket

host = '10.10.10.10'
port = 50000
backlog = 5
size = 1024
s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
s.bind((host,port))
s.listen(backlog)
while 1:
    client, address = s.accept()
    data = "Return data\n"
    client.send(data)
    client.close()

#!/bin/python

import socket

host = '10.10.10.10'

port = 50000

backlog = 5

size = 1024

s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)

s.bind((host,port))

s.listen(backlog)

while 1:

client, address = s.accept()

data = "Return data\n"

client.send(data)

client.close()

Create the client connection

#!/bin/python

import socket

host = '10.10.10.10'
port = 50000
backlog = 5
size = 1024
s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
s.bind((host,port))
s.listen(backlog)
while 1:
    client, address = s.accept()
    data = client.recv(size)
    print data
    if data == "\n":
       client.send("junk1\n")
       client.close()
    elif data == "junk\n":
       client.send("junk2\n")
       client.close()
    else:
       client.send("junk3\n")
       client.close()

#!/bin/python

import socket

host = '10.10.10.10'

port = 50000

backlog = 5

size = 1024

s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)

s.bind((host,port))

s.listen(backlog)

while 1:

client, address = s.accept()

data = client.recv(size)

print data

if data == "\n":

client.send("junk1\n")

client.close()

elif data == "junk\n":

client.send("junk2\n")

client.close()

else:

client.send("junk3\n")

client.close()

Reference http://ilab.cs.byu.edu/python/socket/echoserver.html https://pymotw.com/2/socket/tcp.html

Oracle BDA iPython, Notebook and Jupytar Configuration

June 1, 2016July 5, 2017 Eli Kleinman 2 Comments

(No Ratings Yet)

How to install iPython/Notebook on an Oracle BDA Install the below packages

pip install 
ibackports.ssl_match_hostname-3.5.0.1.tar.gz
ipython-1.2.1.tar.gz
pyzmq-15.2.0.zip
tornado-3.2.1.tar.gz

pip install

ibackports.ssl_match_hostname-3.5.0.1.tar.gz

ipython-1.2.1.tar.gz

pyzmq-15.2.0.zip

tornado-3.2.1.tar.gz

add parcels in CDH GUI parcel address

Remote Parcel Repository URLs
https://repo.continuum.io/pkgs/misc/parcels/

1 2	Remote Parcel Repository URLs https://repo.continuum.io/pkgs/misc/parcels/

donwload, distrubite, activate System startup script

/usr/local/notebook/bin/start_notebook.sh
#!/bin/bash

export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:/opt/rh/python27/root/usr/lib64
export PATH=/opt/rh/python27/root/usr/bin:$PATH

# Notebook
export PYSPARK_DRIVER_PYTHON=ipython
export PYSPARK_DRIVER_PYTHON_OPTS="notebook --NotebookApp.open_browser=False --NotebookApp.ip='*' --NotebookApp.port=8880"

# Jupyter
#export PYSPARK_DRIVER_PYTHON=/opt/cloudera/parcels/Anaconda/bin/jupyter
#export PYSPARK_PYTHON=/opt/cloudera/parcels/Anaconda/bin/python
#export PYSPARK_DRIVER_PYTHON_OPTS="notebook --NotebookApp.open_browser=False --NotebookApp.ip='*' --NotebookApp.port=8880"

cd /usr/local/notebook/notebook_data
nohup /usr/bin/pyspark &

/usr/local/notebook/bin/start_notebook.sh

#!/bin/bash

export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:/opt/rh/python27/root/usr/lib64

export PATH=/opt/rh/python27/root/usr/bin:$PATH

# Notebook

export PYSPARK_DRIVER_PYTHON=ipython

export PYSPARK_DRIVER_PYTHON_OPTS="notebook --NotebookApp.open_browser=False --NotebookApp.ip='*' --NotebookApp.port=8880"

# Jupyter

#export PYSPARK_DRIVER_PYTHON=/opt/cloudera/parcels/Anaconda/bin/jupyter

#export PYSPARK_PYTHON=/opt/cloudera/parcels/Anaconda/bin/python

#export PYSPARK_DRIVER_PYTHON_OPTS="notebook --NotebookApp.open_browser=False --NotebookApp.ip='*' --NotebookApp.port=8880"

cd /usr/local/notebook/notebook_data

nohup /usr/bin/pyspark &

source http://www.cloudera.com/documentation/enterprise/latest/topics/spark_ipython.html

Tag: python

Capturing Solaris 11.4 (12) Analytics By Using Remote Administration Daemon (RAD) – Part 5

Creating, Hosting Your Own CoreOS Rkt App Container Images(ACI)

Kubernetes CoreOS Ignition Configuration Generator – Part 6

Python remote backup script

Example Of Capturing IOStat With Python

Capturing Your Own Solaris 11.4 (12) Analytics / Sstore – Part 3

Capturing Your Own Solaris 11.4 (12) Analytics / Sstore – Part 2

How To Create a Python Socket Server

Oracle BDA iPython, Notebook and Jupytar Configuration

About This Blog

COOKIES

Contact Us

POLICIES

Meta