linghtiin/DebugTool
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

积累修改;

Browse Source
This commit is contained in:
何 泽隆
2025-01-20 11:13:57 +08:00
parent 52406c45b9
commit f3847f4f34
12 changed files with 44493 additions and 470 deletions
Download Patch File Download Diff File Expand all files Collapse all files

130
source/data_analysis.py
View File

@@ -7,7 +7,7 @@ from pathlib import Path
from bs4 import BeautifulSoup
import pandas as pd
import matplotlib.pyplot as plt
import matplotlib.dates as mdates
import matplotlib.colors as mcolors
from sqlalchemy import create_engine
from sqlalchemy import MetaData, Table, Column, String, Float, Integer, DateTime
@@ -45,6 +45,20 @@ SemaMap_adapter = {
'curr_out': ('0305121001', 'adapter', True, "输出电流"),
'power_out': ('0305122001', 'adapter', True, "输出功率"),
}
semamap_combiner = {
'IMSI': ('0305102001', 'combiner', False, "IMSI"),
'ICCID': ('0305103001', 'combiner', False, "SIM卡ICCID"),
'MSISDN': ('0305104001', 'combiner', False, "MSISDN"),
'dev_type': ('0305101001', 'combiner', False, "系统类型"),
'facturer': ('0305107001', 'combiner', False, "汇流箱厂家"),
'model': ('0305108001', 'combiner', False, "汇流箱型号"),
'ver_software': ('0305105001', 'combiner', False, "软件版本"),
'ver_hardware': ('0305106001', 'combiner', False, "硬件版本"),
'power_total': ('0305109001', 'combiner', True, "系统总功率"),
'energy_total': ('0305110001', 'combiner', True, "系统累计发电量"),
'energy_daily': ('0305111001', 'combiner', True, "系统日发电量"),
}
SemaMap_meter = {
'mtr_id': ('0305123001', 'meter', False, "电表号"),
'mtr_volt': ('0305124001', 'meter', True, "直流电压"),
@@ -198,8 +212,9 @@ class Lamina_Data(object):
print(f"Get data success, len={len(json_data['data'])}")
table_data = pd.DataFrame(json_data['data'])
column_name = sorted(table_data.columns)
table_data['dev'] = device_id
table_data['time'] = table_data['updateTime'].apply(lambda x: int(time.mktime(time.strptime(x, r"%Y-%m-%d %H:%M:%S"))))
table_data = table_data[['time', *column_name]].drop(columns='updateTime')
table_data = table_data[['time', 'dev', *column_name]].drop(columns='updateTime')
return table_data
else:
print(f"Get data fail, code={json_data['code']}, msg=\n\t{json_data['message']}")
@@ -345,12 +360,13 @@ class Lamina_Data(object):
dev_meter = []
dev_adapter = []
dev_info = []
try:
for dev in sorted(json_data['rows'], key=lambda x: x['devCode']):
print(f"Dev: {dev['devTypeName']}, id={dev['devCode']}")
time.sleep(0.5)
fsu_id = dev['parentCode'] if 'parentCode' in dev.keys() else None
self.get_real_data_by_net(dev['devCode'], fsu_id, header=header)
dev_info.append(self.get_real_data_by_net(dev['devCode'], fsu_id, header=header))
time.sleep(0.5)
match dev['devType']:
case "0101":
@@ -366,6 +382,7 @@ class Lamina_Data(object):
result = {
'result': True,
'station': station_id,
'information': pd.concat(dev_info, ignore_index=True),
'adapter': pd.concat(dev_adapter, ignore_index=True),
'meter': pd.concat(dev_meter, ignore_index=True),
}
@@ -493,8 +510,6 @@ def save_station_by_file2(data_lamina: Lamina_Data, file_path):
file_input = Path(file_path)
file_output = file_input.parent / (file_input.stem + '_output.xlsx')
df_input = pd.read_excel(file_input)
time_start_timestamp = df_input['开始时间'][0].tz_localize('Asia/Shanghai').timestamp()
time_end_timestamp = df_input['结束时间'][0].tz_localize('Asia/Shanghai').timestamp()
if file_output.exists():
finished_station = pd.read_excel(file_output, sheet_name=None)
finished_station["Station"]['station'] = finished_station["Station"]['station'].astype('str')
@@ -506,11 +521,16 @@ def save_station_by_file2(data_lamina: Lamina_Data, file_path):
remain_station = df_input
dataset = []
df_input = df_input.set_index('点位名称')
for name in remain_station['点位名称']:
print(f"Station: {name}")
time_start_timestamp = df_input['开始时间'][name].tz_localize('Asia/Shanghai').timestamp()
time_end_timestamp = df_input['结束时间'][name].tz_localize('Asia/Shanghai').timestamp()
data = data_lamina.spider_station(name, time_start_timestamp, time_end_timestamp)
if data['result']:
dataset.append(data)
analysis_info1(data)
plt.waitforbuttonpress()
elif data['token']:
""" Token 失效 """
data_lamina.api_origin['header']['authorization'] = data['token']
@@ -536,9 +556,99 @@ def save_station_by_file2(data_lamina: Lamina_Data, file_path):
df_meter.to_excel(writer, sheet_name='Meter', index=False, columns=column_meter)
print(f"数据已成功保存到 {file_output}")
return result
def analysis_info(df_station: pd.DataFrame):
""" 站点Log数据分析 """
map_mid = {}
for k, v in SemaMap_adapter.items():
map_mid[v[0]] = v[3]
for k, v in SemaMap_meter.items():
map_mid[v[0]] = v[3]
map_dev = {
'TTE0102': 'Adapter',
'TTE0103': 'Meter',
}
data = df_station.assign(
timestamp = lambda df: pd.to_datetime(df['time'], unit='s', utc=True).apply(lambda x: x.tz_convert('Asia/Shanghai')),
type = lambda df: df['dev'].apply(lambda x: map_dev[x[:7]]),
date = lambda df: df['timestamp'].apply(lambda x: x.date()),
name = lambda df: df['mid'].map(map_mid),
value = lambda df: pd.to_numeric(df['value'])
)
data_daliy = data.loc[(data['dev'] == 'TTE0102DX2406272727') & (data['date'] == np.datetime64('2024-12-25')) & (data['type'] == 'Adapter')]
fig, axes = plt.subplots(3, 2)
axes = axes.flatten()
i = 0
for name, df_plot in data_daliy.set_index('timestamp').sort_index()[['name', 'value']].groupby('name'):
df_plot.plot(ax=axes[i], title=name)
i += 1
plt.show()
def analysis_info1(data_station: dict):
""" 站点spider返回数据分析 """
# 创建双色颜色过渡
color_map = mcolors.LinearSegmentedColormap.from_list("mycmap", ["blue", "red"])
for dev_id in data_station['information']['dev'].unique():
data_dev = data_station['information'].loc[data_station['information']['dev'] == dev_id]
print(f"Device: {dev_id}")
match dev_id[:7]:
case "TTE0101": # 汇流箱
pass
case "TTE0102": # 适配器
pass
history_dev = data_station['adapter'].assign(
date = lambda df: df['time'].apply(lambda x: x.date()),
)
case "TTE0103": # 电表
pass
history_dev = data_station['meter'].assign(
date = lambda df: df['time'].apply(lambda x: x.date()),
id_group = lambda df: df['date'].diff().ne(0).cumsum(),
)
# 按日期分组并绘制折线图
fig, axs = plt.subplots(3, 1)
axs = axs.flatten()
for date, group in history_dev.groupby('date'):
# 计算当天的起始时间
start_time = pd.Timestamp(date)
# 调整时间索引，使其从当天的起始时间开始
adjusted_time = group['time'] - start_time
# 计算颜色和不透明度
color = color_map(group['id_group'] / history_dev['id_group'][-1])
alpha = 0.5
group.set_index(adjusted_time)['volt'].plot(ax=axs[0], label=str(date), color=color, alpha=alpha)
group.set_index(adjusted_time)['curr'].plot(ax=axs[1], label=str(date), color=color, alpha=alpha)
group.set_index(adjusted_time)['power'].plot(ax=axs[2], label=str(date), color=color, alpha=alpha)
# 添加图例
axs[0].legend(title='Date')
# 添加标题和标签
axs[0].set_title('Value over Time by Date')
axs[0].set_xlabel('Timestamp')
axs[0].set_ylabel('Value')
plt.show()
plt.savefig(Path(f"result\Analysis\{dev_id}.png"))
print(data_dev.head())
if __name__=='__main__':
""" 主体调用流程 """
# plt中文显示
plt.rcParams['font.sans-serif'] = ['SimHei']
# 坐标轴负数显示
plt.rcParams['axes.unicode_minus'] = False
if not hasattr(__builtins__,"__IPYTHON__") and 0:
import pickle
path_data1 = Path(r"result\Analysis\station_data1.pkl")
with open(path_data1, 'rb') as f:
loaded_data = pickle.load(f)
analysis_info1(loaded_data)
if hasattr(__builtins__,"__IPYTHON__"):
path_db = '../result/chinatowercom.db'
else:
@@ -547,22 +657,22 @@ if __name__=='__main__':
if not (file_db:= Path(path_db)).exists():
file_db.touch()
API_HEADER['Cookie'] = "HWWAFSESID=7e7df7972959068f88; HWWAFSESTIME=1736230263315; dc04ed2361044be8a9355f6efb378cf2=WyIzNTI0NjE3OTgzIl0"
API_HEADER['authorization'] = 'Bearer eyJhbGciOiJSUzI1NiIsInR5cCI6IkpXVCJ9.eyJhdWQiOlsiIl0sInVzZXJfbmFtZSI6IndlYl9tYW5hZ2V8d2FuZ2xlaTQiLCJzY29wZSI6WyJhbGwiXSwiZXhwIjoxNzM2MjM3NTMxLCJ1c2VySWQiOjI0Mjg1LCJqdGkiOiIzMWRlNDRiMy05ZTNjLTQwOTEtOWUzMS0wYWFjNTYzZDljZWIiLCJjbGllbnRfaWQiOiJ3ZWJfbWFuYWdlIn0.INV4DumSZkZZ68TW0DTF1XlIIFuoHD90_JefmmOBxcHPDxHAZPzG4JX9BEcEPrRLfSENtfYW7XNCluzB9nxs_pBTT9iu--tPZwlLAiPD7LZ552VdoAFEsYaigFmwxtedTLTzzm2GVbUReInd1dARjgaK0mKxljkKfGkTJURobpHC9Aw5mu25fSWjv7U9sZ0gOmpCuFr_OukEssi0hV8lvztfN5Ax_E1NObbteY2e8tUh6xVj49pHwDPnQScofGTaSviuMO46zmim6X3AKUJ-jDa95dOygKhk704AiA2nVCHXrlVkJI7zYLZB_zZTw3EhyonpksYS8NPp9wLlearWqg'
API_HEADER['Cookie'] = "HWWAFSESTIME=1737167522632; HWWAFSESID=6cb0288b7bc75e5a66; dc04ed2361044be8a9355f6efb378cf2=WyIzNTI0NjE3OTgzIl0"
API_HEADER['authorization'] = 'Bearer eyJhbGciOiJSUzI1NiIsInR5cCI6IkpXVCJ9.eyJhdWQiOlsiIl0sInVzZXJfbmFtZSI6IndlYl9tYW5hZ2V8d2FuZ2xlaTQiLCJzY29wZSI6WyJhbGwiXSwiZXhwIjoxNzM3MzQxNDg4LCJ1c2VySWQiOjI0Mjg1LCJqdGkiOiIwOGFlZDdjYy1hZGE2LTQ4ZWQtYmQyZS0xYjY3NGRkZmVmMWMiLCJjbGllbnRfaWQiOiJ3ZWJfbWFuYWdlIn0.CnfJh2ie0D0dOG1yELiQPuXCwez_nzeYD8rXTL0ILSeq31kmTnhOJJTA6aI8JTEtDVgFyqC084uDR1KvDgwKL5aXXzKwCNqBxziJQbA2AuBRdDgdWXM0r_3qrBGL-0MuYB2jygJaNwue2GIh_3PYsMQGRqHBeyJ9JUgdiWYUVpmbYInSyOlY2l_QtzQTFlz8L7eUC0sDeAWSPNamvYczLas0MtuQquH6JM_-WaFfc-6TblmFp6qSxZHJT-0dy7LLTw5zpXbh3QnbjgBARCaOvzLaDtfArgU20Hq3AqAIwvTVOQFeI4jChFIRvyXwnnUDX-IrFru_sOYLX1jcc88cPA'
data_lamina = Lamina_Data('sqlite:///' + path_db)
# 依据站点内设备爬取整个站点的实时与历史数据
# today = datetime.datetime.today()
# yesterday = today - datetime.timedelta(days=1)
# yesterday = today - datetime.timedelta(days=30)
# today_midnight = today.replace(hour=0, minute=0, second=0, microsecond=0)
# yesterday_midnight = yesterday.replace(hour=0, minute=0, second=0, microsecond=0)
# today_midnight_timestamp = time.mktime(today_midnight.timetuple())
# yesterday_midnight_timestamp = time.mktime(yesterday_midnight.timetuple())
# data = data_lamina.spider_station('TTE0102DX2410091439', yesterday_midnight_timestamp, today_midnight_timestamp)
# data = data_lamina.spider_station("乐亭后庞河村", yesterday_midnight_timestamp, today_midnight_timestamp)
# 读取站点历史数据
# save_station_by_file1(data_lamina)
save_station_by_file2(data_lamina, "result\station_Q0107.xlsx")
result = save_station_by_file2(data_lamina, "result\station_Q0120.xlsx")
# 网站令牌更新
body = {