标签：name parent filename append 数据处理 table columns

import re
from pathlib import Path

import pandas as pd
import sys
import os
def clean(paths):
    return_data = []
    with open(paths,"r") as f:
        text = f.read()

    lists = []
    strs = []
    count = 0
    for i in text.split("\n"):
        # print(i)
        if "DataName" in i :
            if strs:
                lists.append(strs)
            strs = []
            i = '%s,'%count + i.replace("  ",",").replace(" ",",")
            strs.append(i)
            # lists.append(i.split(","))
            # lists.append(str(count))

        if "DataValue" in i:
            i = '%s,'%count + i.replace(" ",",")
            strs.append(i)
            # lists.append(i.split(","))
            # lists.append(str(count))
        count +=1
    else:
        if strs:
            lists.append(strs)
    for i in lists:
        table = pd.DataFrame([ii.split(',') for ii in i]).applymap(lambda x:"" if x == None else x)
        # print(table)
        columns_name = []
        columns = table.columns
        for index, row in table.iterrows():
            if index == 0:
                # 提取Dataname行数据
                for c in columns.values.tolist():
                    columns_name.append(row[c])
            # print(columns_name)
            if index > 1:
                # 提取Datavalue行数据
                for c in range(2,len(columns_name)):
                    # 判断是否为空值
                    if row[c]:
                        return_data.append([columns_name[c],row[c],row[0]])

    # print(return_data)
    return return_data

def fileclean(inputdirs,outputfile):
    datas = []
    for folderName,subfolders,filenames in os.walk(inputdirs):
        # print(folderName,subfolders,filenames)
        for filename in filenames:
            if filename.endswith(".csv"):
                parent = folderName.split("\\")[-1]
                paths = folderName+'/'+filename
                parent_s = parent.split("_")
                Devices_time = re.findall(";(.*?)]",filename)[0].replace("_",'/',2).replace("_",":")
                test_name = filename.split(" ")[0]
                Wafer_id = re.findall("\[(.*?)-",filename)[0]
                test_pipeline = re.findall("-(.*?)\(",filename)[0]
                #ICES5V [A213911020-T1D1(17) ; 11_7_2021 10_31_26 PM]
                for i,j,k in clean(paths=paths):
                    data = {
                        "PATH":parent,
                        "FILE_NAME":filename,
                        "DATANAME":i,
                        "DATAVALUE":j,
                        "PROJECT_TYPE":parent_s[0],
                        "PRODUCT":parent_s[1],
                        "PRODUCT_VERSION":parent_s[2],
                        "LOT":parent_s[3],
                        "TEST_ITEM":parent_s[4],
                        "TEST_NODE":parent_s[5],
                        "FACTORY":parent_s[6],
                        "DEVICES_TIME":Devices_time,
                        "TEST_NAME":test_name,
                        "WAFER_ID":Wafer_id,
                        "TEST_PIPELINE":test_pipeline,
                        "LINE_NO":k
                    }
                    datas.append(data)
        if datas:
            table = pd.DataFrame(datas)
            # 日期处理
            table['DEVICES_TIME'] = pd.to_datetime(table['DEVICES_TIME'])
            table.to_csv(outputfile,index=None)
if __name__ == '__main__':
    argv = sys.argv
    print(argv)
    fileclean(argv[1],argv[2])
    # python clean_data.py 文件夹路径 输出文件路径.csv
    # fileclean("data","data2.csv")

标签：name,parent,filename,append,数据处理,table,columns
来源： https://blog.csdn.net/qq_32888845/article/details/122447035

本站声明： 1. iCode9 技术分享网（下文简称本站）提供的所有内容，仅供技术学习、探讨和分享；
2. 关于本站的所有留言、评论、转载及引用，纯属内容发起人的个人观点，与本站观点和立场无关；
3. 关于本站的所有言论和文字，纯属内容发起人的个人观点，与本站观点和立场无关；
4. 本站文章均是网友提供，不完全保证技术分享内容的完整性、准确性、时效性、风险性和版权归属；如您发现该文章侵犯了您的权益，可联系我们第一时间进行删除；
5. 本站为非盈利性的个人网站，所有内容不会用来进行牟利，也不会利用任何形式的广告来间接获益，纯粹是为了广大技术爱好者提供技术内容和技术思想的分享性交流网站。