🕷️ 数据采集与处理在线学习

# 简单的正则表达式模拟 def match_pattern(pattern, text): """模拟正则匹配""" if pattern == r'\d+': return ''.join([c for c in text if c.isdigit()]) elif pattern == r'\w+': return ''.join([c for c in text if c.isalnum() or c == '_']) return "" # 测试匹配 text = "张三的电话是13812345678" print(f"原文：{text}") numbers = match_pattern(r'\d+', text) print(f"提取数字：{numbers}") text2 = "Hello_World123" words = match_pattern(r'\w+', text2) print(f"提取单词：{words}")

# 模拟正则提取操作 def extract_emails(text): """提取邮箱地址""" result = [] i = 0 while i < len(text): if text[i] == '@': # 向前找用户名 start = i while start > 0 and (text[start-1].isalnum() or text[start-1] == '_' or text[start-1] == '.'): start -= 1 # 向后找域名 end = i while end < len(text) and (text[end].isalnum() or text[end] == '@' or text[end] == '.' or text[end] == '_'): end += 1 email = text[start:end] if '@' in email and email.count('@') == 1: result.append(email) i = end else: i += 1 return result # 测试提取 text = "联系邮箱：test@example.com 或 admin@test.org" print(f"原文：{text}") emails = extract_emails(text) print(f"提取的邮箱：{emails}")

# 模拟CSV数据解析 def parse_csv(csv_text): """解析CSV文本""" lines = csv_text.strip().split('\n') headers = lines[0].split(',') data = [] for line in lines[1:]: values = line.split(',') if len(values) == len(headers): data.append(dict(zip(headers, values))) return data # 模拟CSV数据 csv_data = """姓名,年龄,城市 张三,25,北京 李四,30,上海 王五,28,广州 赵六,22,深圳""" print("CSV数据解析结果：") print("="*40) records = parse_csv(csv_data) for record in records: print(f" {record['姓名']}: {record['年龄']}岁, 来自{record['城市']}") print("="*40) print(f"共 {len(records)} 条记录")

# 模拟JSON解析 def parse_json(json_str): """简化版JSON解析""" return eval(json_str) # 模拟JSON数据 json_data = """{ "姓名": "张三", "年龄": 25, "技能": ["Python", "数据分析", "机器学习"], "项目": { "名称": "销售分析系统", "周期": "3个月" } }""" print("JSON数据解析结果：") print("="*40) data = parse_json(json_data) print(f"姓名：{data['姓名']}") print(f"年龄：{data['年龄']}") print(f"技能：{', '.join(data['技能'])}") print(f"项目：{data['项目']['名称']}") print("="*40)

# 数据清洗示例 raw_data = [85, None, 78, 92, 78, None, 88] print("数据清洗示例：") print("="*40) print(f"原始数据：{raw_data}") # 1. 统计缺失值 null_count = raw_data.count(None) print(f"缺失值数量：{null_count}") # 2. 用平均值填充缺失值 valid_nums = [x for x raw_data if x is not None] avg = sum(valid_nums) / len(valid_nums) filled_data = [avg if x is None else x for x raw_data] print(f"填充后数据：{[round(x, 1) for x in filled_data]}") # 3. 去重 unique_data = list(dict.fromkeys(filled_data)) print(f"去重后数据：{[round(x, 1) for x in unique_data]}") print("="*40)

# 数据转换示例 # 原始数据（字符串） raw_numbers = ["85", "92", "78", "95", "88"] print("数据转换示例：") print("="*40) print(f"原始数据（字符串）：{raw_numbers}") # 1. 类型转换：str → int numbers = [int(x) for x raw_numbers] print(f"转换为整数：{numbers}") # 2. 归一化处理（0-1范围） min_val = min(numbers) max_val = max(numbers) normalized = [(x - min_val) / (max_val - min_val) for x numbers] print(f"归一化后：{[round(x, 2) for x in normalized]}") # 3. 数据分组 grade_map = lambda x: "A" if x >= 90 else ("B" if x >= 80 else "C") grades = [grade_map(x) for x numbers] print(f"成绩等级：{grades}") print("="*40)

# 数据验证与汇总示例 sales_data = [ {"日期": "2024-01", "销售额": 12000, "产品": "A"}, {"日期": "2024-01", "销售额": 15000, "产品": "B"}, {"日期": "2024-02", "销售额": 18000, "产品": "A"}, {"日期": "2024-02", "销售额": 14000, "产品": "B"}, ] print("销售数据分析：") print("="*40) # 1. 数据验证 print("数据验证：") for record in sales_data: valid = True if record["销售额"] < 0: valid = False if not record["日期"]: valid = False print(f" {record['日期']} {record['产品']}: {'✓ 有效' if valid else '✗ 无效'}") # 2. 按月份汇总 print("\n按月份汇总：") monthly = {} for r in sales_data: month = r["日期"] monthly[month] = monthly.get(month, 0) + r["销售额"] for month, total in monthly.items(): print(f" {month}: ¥{total:,}") # 3. 总体统计 total_sales = sum(r["销售额"] for r in sales_data) avg_sales = total_sales / len(sales_data) print(f"\n总体统计：") print(f" 总销售额：¥{total_sales:,}") print(f" 平均单笔：¥{avg_sales:,.0f}") print("="*40)