Comcast Modem Power Level Monitor

@echo off

wsl -u root service ssh start
wsl -u root service cron start

exit

0 * * * * /home/user/scripts/comcast/cm_run.sh
*/1 * * * * /home/user/scripts/comcast/keep_alive.sh

#!/bin/bash

~/scripts/comcast/comcastplot.R

#!/bin/bash

if ! pgrep -f 'cmpwrlevel2.py'
then
    nohup $(which python3) ~/scripts/comcast/cmpwrlevel2.py &
fi

2019-09-16 01:30:17 U 43.78 dBmV D 3.97 dBmV
2019-09-16 01:31:18 U 43.78 dBmV D 3.97 dBmV
2019-09-16 01:32:18 U 43.78 dBmV D 3.97 dBmV
2019-09-16 01:33:18 U 43.78 dBmV D 3.96 dBmV
2019-09-16 01:34:19 U 43.78 dBmV D 3.96 dBmV
2019-09-16 01:35:18 U 43.78 dBmV D 4.03 dBmV
2019-09-16 01:36:18 U 43.78 dBmV D 4.00 dBmV
2019-09-16 01:37:18 U 43.78 dBmV D 3.98 dBmV
2019-09-16 01:38:17 U 43.78 dBmV D 3.97 dBmV

#!/usr/bin/python3

# Comcast modem power levels reading script.
# Author: Eugenio C.
# Date: 09.16.2019

# TODO: Rewrite this script into a module with calls to objects.
# TODO: Put raw data in database. SQLite

# Load libraries
import logging
import re
import smtplib
import datetime
import pickle
import pathlib
import os
import sys
import time
import statistics
from email.message import EmailMessage
from bs4 import BeautifulSoup as bs
import requests

DATAFILE = 'cm_modem_pwr_levels.txt'
WAIT_TIME = 60

# ********************************************************************************
# Function Declerations
# ********************************************************************************

def send_text(msg_txt):
    msg = EmailMessage()

    msg.set_content(msg_txt)

    msg['Subject'] = 'CM Alarm'
    msg['From'] = 'python_script@yourdomain.com'
    msg['To'] = 'phonenumber@vtext.com'

    smtp = smtplib.SMTP('email.server.address')
    smtp.send_message(msg)
    smtp.quit()

# ********************************************************************************
# Main Script
# ********************************************************************************

ownp = os.path.abspath(os.path.dirname(sys.argv[0]))
tdnow = datetime.datetime.now().strftime("%Y%m%d")

# Setup Logging
logging.basicConfig(
    format='%(asctime)s:%(levelname)s:%(message)s',
    level=logging.DEBUG,
    filename=ownp + '/cmpwrlevel2.log')


# Get our "browser"
s = requests.Session()

# Reload cookies from previous session if they exist
if pathlib.Path(ownp + '/cookiefile2').exists():
    with open(ownp + '/cookiefile2', 'rb') as f:
        s.cookies.update(pickle.load(f))

# Set url and authentication
url = "http://10.1.10.1/check.php"
payload = {
    "username": "cusadmin", # Public, well known, customer-side, default credentials
    "password": "highspeed"
}

# Set our regex
regex = r'"Please Login First\!\"'
modemin = False

# Login loop, tries to connect to the modem.
while not modemin:
    try:
        # Login to set cookie
        logging.info("Getting: %s" % url)
        r = s.post(url, data=payload)

        logging.info("Status Code: %i" % r.status_code)
        r.raise_for_status()

        if not re.search(regex, str(r.content)):
            logging.info("Ok, we're in. Let's stay logged on and get data!")

            send_text("Logged on!")

            modemin = True
        else:
            logging.info("No Luck! Let's try again in 10 seconds.")
            time.sleep(10)

    except Exception as e:
        logging.error(e)

# Save cookies
with open(ownp + '/cookiefile2', 'wb') as f:
    pickle.dump(s.cookies, f)

# Now we can access the page we need
url = "http://10.1.10.1/comcast_network.php"

getdata = True

# Let's start the data loop
while getdata:
    try:
        logging.info("Getting: %s" % url)
        r = s.get(url)

        logging.info("Status Code: %i" % r.status_code)
        r.raise_for_status()

        # Get the date/time close to the time we read the values
        currentDT = datetime.datetime.now()

        # Get the html
        bs_content = bs(r.content, "lxml")

        # Narrow the html down to the two tables
        down_data = bs_content.find_all('table')[0].get_text()
        up_data = bs_content.find_all('table')[1].get_text()

        # Set our regex
        regex = r"[-0-9\.]* dBmV"

        # Get the power levels
        d_matches = re.finditer(regex, down_data, re.MULTILINE)
        u_matches = re.finditer(regex, up_data, re.MULTILINE)

        d_pwr_sum = []
        u_pwr_sum = []

        # For each power level remove the label and convert to a number
        # Sum up all the power levels so we can divide later for the average
        for d_matchNum, match in enumerate(d_matches, start=1):
            d_pwr = float(match.group().replace(" dBmV", ""))
            d_pwr_sum.append(d_pwr)

        for u_matchNum, match in enumerate(u_matches, start=1):
            u_pwr = float(match.group().replace(" dBmV", ""))
            u_pwr_sum.append(u_pwr)

        # Print info to log file
        logging.info("DW: {}".format(','.join(map(str, d_pwr_sum))))
        logging.info("DW: H%.1f, L%.1f, M%.1f, A%.2f" %
                     (max(d_pwr_sum),
                      min(d_pwr_sum),
                      statistics.median(d_pwr_sum),
                      statistics.mean(d_pwr_sum)))

        logging.info("UP: {}".format(','.join(map(str, u_pwr_sum))))
        logging.info("UP: H%.1f, L%.1f, M%.1f, A%.2f" %
                     (max(u_pwr_sum),
                      min(u_pwr_sum),
                      statistics.median(u_pwr_sum),
                      statistics.mean(u_pwr_sum)))

        # Calculate the averages
        avg_u_pwr = statistics.mean(u_pwr_sum)
        avg_d_pwr = statistics.mean(d_pwr_sum)

        # If the power levels are outside of spec, send an alert text message
        if (avg_u_pwr > 49) or (avg_u_pwr < 40) or (avg_d_pwr > 10) or (avg_d_pwr < -10):
            send_text("U%.2f D%.2f" % (avg_u_pwr, avg_d_pwr))

        # Write data to file
        logging.info("Writing data to: %s" % DATAFILE)
        with open(DATAFILE, 'a') as o:
            o.write(
                "%s U %.2f dBmV D %.2f dBmV\n" %
                (currentDT.strftime("%Y-%m-%d %H:%M:%S"),
                 avg_u_pwr,
                 avg_d_pwr))

        logging.info("Waiting %i seconds." % (WAIT_TIME))
        time.sleep(WAIT_TIME)

    except Exception as e:
        logging.error(e)

        send_text("Exception!")

        with open(ownp + '/content.html', 'w+') as h:
            h.write(r.text)

        # Exit loop, cron will restart
        getdata = False

#!/usr/bin/env Rscript

# Comcast modem power levels plotting script.
# Author: Eugenio Cilento
# Date: 09.16.2019

# Load libraries
library(ggplot2, warn.conflicts = FALSE)
library(scales, warn.conflicts = FALSE)
library(ggrepel, warn.conflicts = FALSE)
library(lubridate, warn.conflicts = FALSE)
library(dplyr, warn.conflicts = FALSE)

consts <- list(img_w = 14,
    img_h = 8.5,
    img_dpi = 320,
    img_q = 30,
    days = 6)

# ***************************************************************************************
# Get the data, clean it up and get it ready for use in the graph.
# ***************************************************************************************

# Read in the modem power levels from the text file.
pwr <- read.table("cm_modem_pwr_levels.txt")

pwr = pwr[c(-3,-5,-6,-8)] # Remove uneeded columns.

colnames(pwr) = c("Date","Time","UP","DW") # Let's give the columns some names.

pwr$DateTime <- paste(pwr$Date, pwr$Time) # Create a new timstamp column.

# Create a new column used for the facets. Basically to show the Days on the graph.
pwr$Dow <- strptime(pwr$DateTime, "%Y-%m-%d")
pwr$Dow <- format(pwr$Dow, "%m-%d %a")

pwr = pwr[c(-1,-2)] # Some more cleanup.

# Make that column an actual date time field
pwr$DateTime <- strptime(pwr$DateTime, "%Y-%m-%d %H:%M:%S")
pwr$DateTime <- as.POSIXct(pwr$DateTime, tz = "America/Chicago") # Convert to POSIXct type

pwr <- pwr[pwr$DateTime >= as.POSIXct(Sys.Date()-consts$days),] # Limit the data.

# When did the modem signal go outside of the acceptable range?
pwrovr <- pwr[which(pwr[,1] > 49 | pwr[,1] < 35 ),]

# Dataframe for ranges
dmax <- pwr %>% group_by(Dow) %>% summarise(Max = max(DateTime))
dmin <- pwr %>% group_by(Dow) %>% summarise(Min = min(DateTime))
dminmax <- merge(dmax,dmin,by = "Dow")

# Put in variables so we dont have to repeat code
dta = as.POSIXct(Sys.Date())
dtw = format(as.POSIXct(Sys.Date()), "%m-%d %a")

# ***************************************************************************************
# Beginning of the UPSTREAM graph script.
# ***************************************************************************************

# Dataframe for text labels for ranges
text_ann <- data.frame(UP = 49, DW = 5,
  lab = "Packet Loss\nBegins: +49", DateTime = dta, Dow = dtw)

text_ann <- rbind(text_ann, 
  data.frame(UP = 35, DW = 5, 
    lab = "Recommended:\n+ 35 - 47", DateTime = dta, Dow = dtw))

hour(text_ann$DateTime) = 2 #position of text within facet

# Create the plot
gg <- ggplot(pwr, aes(DateTime, UP)) +
  geom_point( # Points on a graph seem to work best for this type data.
    color = "black",
    size = .1
  ) + 
  facet_grid( # This gives the days in a nice column.
    col = vars(Dow),
    labeller = labeller(Dow = label_wrap_gen(5)), # Word rap the labels.
    space = 'free', 
    scales = 'free', 
    switch = 'x'
  ) +
  labs( # Needs some good titles and labels to make it easier to understand.
    title="Cable Modem Upstream Power", 
    subtitle="Comcast Business Internet", 
    y="Upstream Power Level (dBmV)", 
    x="Time (Date & Hour in 24Hr format)", 
    caption= paste( # Timestamp the graph.
      "International Assurance of TN, Inc. IT Department -",
      Sys.time()
    )
  ) +
  geom_rect( # color the acceptable range so it stands out.
    data = dminmax, 
    aes(
      xmin = Min, 
      xmax = Max, 
      ymin = 35, 
      ymax = 47), 
    alpha = 0.2, 
    fill="green", 
    inherit.aes = FALSE
  ) +
  geom_rect( # color the acceptable range so it stands out.
    data = dminmax, 
    aes(
      xmin = Min, 
      xmax = Max, 
      ymin = 47, 
      ymax = 49), 
    alpha = 0.2, 
    fill="red", 
    inherit.aes = FALSE
  ) +
  geom_text( # add labels for the ranges
    data = text_ann, 
    aes(x=DateTime, y=UP, label=lab), 
    vjust=-.2,
    hjust=0,
    size=2.5,
    color= "black"
  ) +
  geom_hline(yintercept=49, linetype="dashed", color = "red") +
  geom_hline(yintercept=47, linetype="dashed", color = "green") +
  geom_hline(yintercept=35, linetype="dashed", color = "green") +
  coord_cartesian( # Set the y scale. The range of modem signal levels.
    ylim = c(0, 60)
  ) +
  scale_y_continuous( # Set where the lines show on the graph.
    minor_breaks = seq(0 , 60, 1), 
    breaks = seq(0, 60, 10)
  ) +
  scale_x_datetime( # Set the x scale based on the hour of the day.
    breaks = pretty_breaks(), # Makes the hours fit nicely in a day.
    expand = c(0, 0), # Brings the panel/facets together, seamless transition from day to day.
    labels = date_format("%H", tz = Sys.timezone(location = TRUE)), 
    date_breaks = "2 hour",
    date_minor_breaks = "1 hour"
  ) + 
  theme_bw() + # Start off with a base theme, then modify it.
  theme(
    panel.border = element_rect(
      color = "dark gray"),
    panel.spacing.x = unit(0,"line"),
    strip.placement = "outside",
    strip.text.x = element_text(
      size = 6
    ),
    panel.grid.major = element_line(
      color = "light gray"),
    panel.grid.minor = element_line(
      color = "light gray"
    ),
    axis.text.x = element_text( # Hour of the day
      size = 6,
      angle = 90,
      vjust = 0.5,
      hjust = 0.5),
    axis.line = element_line( # Make the axis stand out.
      color = "darkblue", 
      size = 1, 
      linetype = "solid")
  ) 

if (dim(pwrovr)[1] != 0) { # Make sure we have out of range plots before adding them 
  gg = gg +
    geom_text_repel( # Label the out-of-range plots
      aes(label = paste(hour(DateTime),":", minute(DateTime),"-",UP)), 
      pwrovr,
      size = 2,
      color = "red"
    )
}

# Save the graph to a JPEG file. High resolution allows more clarity and printability.
ggsave("cm_modem_up_pwr_levels.jpg", 
       width = consts$img_w, 
       height = consts$img_h, 
       dpi = consts$img_dpi,
       quality = consts$img_q,
       limitsize = FALSE)

# ***************************************************************************************
# Beginning of the DOWNSTREAM graph script.
# ***************************************************************************************

# Dataframe for text labels for  ranges
text_ann2 <- data.frame(UP = -6, DW = 5,
  lab = "Recommended:\n-7 to 7", DateTime = dta, Dow = dtw)

text_ann2 <- rbind(text_ann2, 
  data.frame(UP = -9, DW = 5, 
    lab = "Acceptable:\n+- 7 to 10", DateTime = dta, Dow = dtw))

text_ann2 <- rbind(text_ann2, 
  data.frame(UP = -14, DW = 5, 
    lab = "Maximum:\n+- 10 to 15 ", DateTime = dta, Dow = dtw))

text_ann2 <- rbind(text_ann2, 
  data.frame(UP = -18, DW = 5, 
    lab = "Out of Spec:\n> +-15 ", DateTime = dta, Dow = dtw))

hour(text_ann2$DateTime) = 2 #position of text within facet

# Filter into a new table the out-of-range instances.
pwrovr <- pwr[which(pwr[,2] > 10 | pwr[,2] < -10 ),]

# Generate the downstream graph. Identical to upstream but with different ranges.
gg <- ggplot(pwr, aes(DateTime, DW)) +
  geom_point(
    color = "black",
    size = .1
  ) +
  facet_grid(
    col = vars(Dow),
    labeller = labeller(Dow = label_wrap_gen(5)),
    space = 'free', 
    scales = 'free', 
    switch = 'x'
  ) +
  coord_cartesian(
    ylim = c(-30, 20)
  ) +
  scale_y_continuous(
    minor_breaks = seq(-30 , 20, 1), 
    breaks = seq(-30, 20, 10)
  ) +
  labs(
    title="Cable Modem Downstream Power",
    subtitle="Comcast Business Internet",
    y="Downstream Power Level (dBmV)",
    x="Time (Date & Hour in 24Hr format)",
    caption= paste(
      "International Assurance of TN, Inc. IT Department -",
      Sys.time()
    )
  ) +
  geom_text( # write the labels for the ranges
    data = text_ann2, 
    aes(x=DateTime, y=UP, label=lab), 
    vjust=.3,
    hjust=0,
    size=2.5,
    color= "black"
  ) +
  geom_hline(yintercept=-15, linetype="dashed", color = "red") +
  geom_hline(yintercept=-10, linetype="dashed", color = "yellow") +
  geom_hline(yintercept=-7, linetype="dashed", color = "green") +
  geom_hline(yintercept=7, linetype="dashed", color = "green") +
  geom_hline(yintercept=10, linetype="dashed", color = "yellow") +
  geom_hline(yintercept=15, linetype="dashed", color = "red") +
  geom_rect( # color the acceptable range so it stands out.
    data = dminmax, 
    aes(
      xmin = Min, 
      xmax = Max, 
      ymin = -7, 
      ymax = 7), 
    alpha = 0.2, 
    fill="green", 
    inherit.aes = FALSE
  ) +
  geom_rect( # color the acceptable range so it stands out.
    data = dminmax, 
    aes(
      xmin = Min, 
      xmax = Max, 
      ymin = -10, 
      ymax = -7), 
    alpha = 0.2, 
    fill="yellow", 
    inherit.aes = FALSE
  ) +
  geom_rect( # color the acceptable range so it stands out.
    data = dminmax, 
    aes(
      xmin = Min, 
      xmax = Max, 
      ymin = 7, 
      ymax = 10), 
    alpha = 0.2, 
    fill="yellow", 
    inherit.aes = FALSE
  ) +
   geom_rect( # color the acceptable range so it stands out.
    data = dminmax, 
    aes(
      xmin = Min, 
      xmax = Max, 
      ymin = -15, 
      ymax = -10), 
    alpha = 0.2, 
    fill="red", 
    inherit.aes = FALSE
  ) +
  geom_rect( # color the acceptable range so it stands out.
    data = dminmax, 
    aes(
      xmin = Min, 
      xmax = Max, 
      ymin = 10, 
      ymax = 15), 
    alpha = 0.2, 
    fill="red", 
    inherit.aes = FALSE
  ) +
  scale_x_datetime(
    breaks = pretty_breaks(),
    expand = c(0, 0),
    labels = date_format("%H", tz = Sys.timezone(location = TRUE)),
    date_breaks = "2 hour",
    date_minor_breaks = "1 hour"
  ) +
  theme_bw() +
  theme(
    panel.border = element_rect(
      colour = "dark gray"),
    panel.spacing.x = unit(0,"line"),
    strip.placement = "outside",
    strip.text.x = element_text(
      size = 6
    ),
    panel.grid.major = element_line(
      colour = "light gray"),
    panel.grid.minor = element_line(
      color = "light gray"
    ),
    axis.text.x = element_text(
      size = 6,
      angle = 90,
      vjust = 0.5,
      hjust = 0.5),
    axis.line = element_line(
      colour = "darkblue", 
      size = 1, 
      linetype = "solid")
  )

if (dim(pwrovr)[1] != 0) {  # Make sure we have out of range before plotting
  gg = gg +
    geom_text_repel( # Label the out-of-range plots
      aes(label = paste(DateTime, "-", DW, "dBmV")),
      pwrovr,
      size = 2,
      color = "red"
    )
}

# Save graph to file.
ggsave("cm_modem_dwn_pwr_levels.jpg", 
       width = consts$img_w, 
       height = consts$img_h, 
       dpi = consts$img_dpi,
       quality = consts$img_q,
       limitsize = FALSE)