Ad

PfSense Python Interpreter Doen't Read Json

- 1 answer

I'm scratching my head. My goal is to get an input of json und put it out into an influxdb. Since' I'm gathering data from a pfSense my tools are limited. Therefor I'm using Python 3.8.

Developing on an Debian Machine first I wrote this code:

#!/usr/local/bin/python3.8
import os
import json
from influxdb import InfluxDBClient
from datetime import datetime

INTERFACES = ["igb0", "pppoe0"]
WORKING_FOLDER = "/tmp/json"
NOW = datetime.now()
CLIENT = InfluxDBClient("host", "8086", "user", "pw", "firewall")
for interface in INTERFACES:
    stats = {}
    os.popen(
        "vnstat --json d 2 -i {} > {}/{}_d.json".format(interface, WORKING_FOLDER, interface)
    ).read()
    os.popen(
        "vnstat --json m 2 -i {} > {}/{}_m.json".format(interface, WORKING_FOLDER, interface)
    ).read()

    with open(f"{WORKING_FOLDER}/{interface}_d.json", "r") as j:
        day = json.loads(j.read())
    with open(f"{WORKING_FOLDER}/{interface}_m.json", "r") as k:
        month = json.loads(k.read())

    if not len(day["interfaces"][0]["traffic"]["day"]) == 2:
        stats["yesterday"] = {"rx": 0, "tx": 0}
        stats["today"] = {
            "rx": day["interfaces"][0]["traffic"]["day"][0]["rx"],
            "tx": day["interfaces"][0]["traffic"]["day"][0]["tx"],
        }

    else:
        stats["yesterday"] = {
            "rx": day["interfaces"][0]["traffic"]["day"][0]["rx"],
            "tx": day["interfaces"][0]["traffic"]["day"][0]["tx"],
        }
        stats["today"] = {
            "rx": day["interfaces"][0]["traffic"]["day"][1]["rx"],
            "tx": day["interfaces"][0]["traffic"]["day"][1]["tx"],
        }

    if not len(month["interfaces"][0]["traffic"]["month"]) == 2:
        stats["last_month"] = {"rx": 0, "tx": 0}
        stats["this_month"] = {
            "rx": month["interfaces"][0]["traffic"]["month"][0]["rx"],
            "tx": month["interfaces"][0]["traffic"]["month"][0]["tx"],
        }

    else:
        stats["last_month"] = {
            "rx": month["interfaces"][0]["traffic"]["month"][0]["rx"],
            "tx": month["interfaces"][0]["traffic"]["month"][0]["tx"],
        }
        stats["this_month"] = {
            "rx": month["interfaces"][0]["traffic"]["month"][1]["rx"],
            "tx": month["interfaces"][0]["traffic"]["month"][1]["tx"],
        }

    json_body = [
        {
            "measurement": f"stats_{interface}",
            "time": NOW,
            "fields": {
                "yesterday_rx": stats["yesterday"]["rx"],
                "yesterday_tx": stats["yesterday"]["tx"],
                "yesterday_total": int(stats["yesterday"]["rx"]) + int(stats["yesterday"]["tx"]),
                "today_rx": stats["today"]["rx"],
                "today_tx": stats["today"]["tx"],
                "today_total": int(stats["today"]["rx"]) + int(stats["today"]["tx"]),
                "last_month_rx": stats["last_month"]["rx"],
                "last_month_tx": stats["last_month"]["tx"],
                "last_month_total": int(stats["last_month"]["rx"]) + int(stats["last_month"]["tx"]),
                "this_month_rx": stats["this_month"]["rx"],
                "this_month_tx": stats["this_month"]["tx"],
                "this_month_total": int(stats["this_month"]["rx"]) + int(stats["this_month"]["tx"]),
            },
        }
    ]

    CLIENT.write_points(json_body)

Testing it on my Debian machine (with line 12 & 13 commented out to not overwrite my json files) everything works like a charm. So I moved the script to my pfSense and I'm getting the following error:

[2.5.2-RELEASE][[email protected]]/usr/local/pythonscripts: ./trafficstats.py
Traceback (most recent call last):
  File "./trafficstats.py", line 18, in <module>
    month = json.loads(k.read())
  File "/usr/local/lib/python3.8/json/__init__.py", line 357, in loads
    return _default_decoder.decode(s)
  File "/usr/local/lib/python3.8/json/decoder.py", line 337, in decode
    obj, end = self.raw_decode(s, idx=_w(s, 0).end())
  File "/usr/local/lib/python3.8/json/decoder.py", line 355, in raw_decode
    raise JSONDecodeError("Expecting value", s, err.value) from None
json.decoder.JSONDecodeError: Expecting value: line 1 column 1 (char 0)

And I'm asking myself why. Because looking at the moth file is a valid json if I see correctly:

{"vnstatversion":"2.7","jsonversion":"2","interfaces":[{"name":"igb0","alias":"WAN","created":{"date":{"year":2022,"month":2,"day":4}},"updated":{"date":{"year":2022,"month":2,"day":12},"time":{"hour":9,"minute":30}},"traffic":{"total":{"rx":82416467756,"tx":43825701833},"month":[{"id":88,"date":{"year":2022,"month":2},"rx":82416467756,"tx":43825701833}]}}]}

Does anybody got a hint or idea whats going on?

Cheers, Gamie

Ad

Answer

"Expecting value at char 0" generally means that you're trying to json.loads() an empty string:

>>> import json
>>> json.loads("")
Traceback (most recent call last):
  File "<stdin>", line 1, in <module>
  File "C:\Python39\lib\json\__init__.py", line 346, in loads
    return _default_decoder.decode(s)
  File "C:\Python39\lib\json\decoder.py", line 337, in decode
    obj, end = self.raw_decode(s, idx=_w(s, 0).end())
  File "C:\Python39\lib\json\decoder.py", line 355, in raw_decode
    raise JSONDecodeError("Expecting value", s, err.value) from None
json.decoder.JSONDecodeError: Expecting value: line 1 column 1 (char 0)

You're not checking whether those vnstat invocations succeed (you should be using subprocess.check_call() instead of os.popen anyway since you're not really even using pipes), so it's likely the files have been created but are empty.

All in all, I'd maybe refactor things to something like this:

  • use subprocess.check_call() to wait for the process to finish (and check result)
  • don't use a stats dict that's basically just thrown away
  • refactor repetitive bits
#!/usr/local/bin/python3.8
import json
import subprocess
from datetime import datetime

from influxdb import InfluxDBClient

INTERFACES = ["igb0", "pppoe0"]
WORKING_FOLDER = "/tmp/json"
NOW = datetime.now()
CLIENT = InfluxDBClient("host", "8086", "user", "pw", "firewall")


# Process an interfaces.X.traffic.PERIOD dict into a pair of (previous_rx, previous_tx), (current_rx, current_tx)
def process_traffic(traffic):
    if len(traffic) != 2:
        return ((0, 0), (traffic[0]["rx"], traffic[0]["tx"]))
    return ((traffic[0]["rx"], traffic[0]["tx"]), (traffic[1]["rx"], traffic[1]["tx"]))


def process_iface(interface):
    day_json = f"{WORKING_FOLDER}/{interface}_d.json"
    month_json = f"{WORKING_FOLDER}/{interface}_m.json"
    # TODO: consider shlex.quote for safety
    subprocess.check_call(
        f"vnstat --json d 2 -i {interface} > {day_json}",
        shell=True,
    )
    subprocess.check_call(
        f"vnstat --json m 2 -i {interface} > {month_json}",
        shell=True,
    )

    with open(day_json, "r") as j:
        day_data = json.loads(j.read())
    with open(month_json, "r") as k:
        month_data = json.loads(k.read())

    (yesterday_rx, yesterday_tx), (today_rx, today_tx) = process_traffic(
        day_data["interfaces"][0]["traffic"]["day"]
    )
    (last_month_rx, last_month_tx), (this_month_rx, this_month_tx) = process_traffic(
        month_data["interfaces"][0]["traffic"]["month"]
    )

    json_body = [
        {
            "measurement": f"stats_{interface}",
            "time": NOW,
            "fields": {
                "yesterday_rx": yesterday_rx,
                "yesterday_tx": yesterday_tx,
                "yesterday_total": int(yesterday_rx) + int(yesterday_tx),
                "today_rx": today_rx,
                "today_tx": today_tx,
                "today_total": int(today_rx) + int(today_tx),
                "last_month_rx": last_month_rx,
                "last_month_tx": last_month_tx,
                "last_month_total": int(last_month_rx) + int(last_month_tx),
                "this_month_rx": this_month_rx,
                "this_month_tx": this_month_tx,
                "this_month_total": int(this_month_rx) + int(this_month_tx),
            },
        }
    ]

    CLIENT.write_points(json_body)


def main():
    for interface in INTERFACES:
        process_iface(interface)

if __name__ == "__main__":
    main()
Ad
source: stackoverflow.com
Ad