准备数据
```python
from pymongo import MongoClientimport datetimeclient=MongoClient('mongodb://localhost:27017')
table=client['db1']['emp']l=[
('张飞','male',18,'20170301','',7300.33,401,1), #以下是教学部('张云','male',78,'20150302','teacher',1000000.31,401,1),('刘备','male',81,'20130305','teacher',8300,401,1),('关羽','male',73,'20140701','teacher',3500,401,1),('曹操','male',28,'20121101','teacher',2100,401,1),('诸葛亮','female',18,'20110211','teacher',9000,401,1),('周瑜','male',18,'19000301','teacher',30000,401,1),('司马懿','male',48,'20101111','teacher',10000,401,1),('袁绍','female',48,'20150311','sale',3000.13,402,2),#以下是销售部门
('张全蛋','female',38,'20101101','sale',2000.35,402,2),('鹌鹑蛋','female',18,'20110312','sale',1000.37,402,2),('王尼玛','female',18,'20160513','sale',3000.29,402,2),('我尼玛','female',28,'20170127','sale',4000.33,402,2),('杨过','male',28,'20160311','operation',10000.13,403,3), #以下是运营部门
('小龙女','male',18,'19970312','operation',20000,403,3),('郭靖','female',18,'20130311','operation',19000,403,3),('黄蓉','male',18,'20150411','operation',18000,403,3),('梅超风','female',18,'20140512','operation',17000,403,3)]for n,item in enumerate(l):
d={ "_id":n, 'name':item[0], 'sex':item[1], 'age':item[2], 'hire_date':datetime.datetime.strptime(item[3],'%Y%m%d'), 'post':item[4], 'salary':item[5] } table.save(d)# 准备数据
```分组的概念与mysql相同,以某个字段作为依据进行归类,其目的是为了统计
## $match
```python
#match 用于对数据进行筛选{"$match":{"字段":"条件"}},可以使用任何常用查询操作符$gt,$lt,$in等#例1、select * from db1.emp where post='teacher';
db.emp.aggregate({"$match":{"post":"teacher"}})#例2、select * from db1.emp where id > 3;
db.emp.aggregate( {"$match":{"_id":{"$gt":3}}},)```## $project
```python
# project翻译为投射 ,即将一个数据结果映射为另一个结果 过程中可以对某些数据进行修改 控制其最终显示的结果{"$project":{"要保留的字段名":1,"要去掉的字段名":0,"新增的字段名":"表达式"}}#1、select name,post,(age+1) as new_age from db1.emp;
db.emp.aggregate( {"$project":{ "name":1, "post":1 }})#2、表达式之数学表达式
{"$add":[expr1,expr2,...,exprN]} #相加{"$subtract":[expr1,expr2]} #第一个减第二个{"$multiply":[expr1,expr2,...,exprN]} #相乘{"$divide":[expr1,expr2]} #第一个表达式除以第二个表达式的商作为结果{"$mod":[expr1,expr2]} #第一个表达式除以第二个表达式得到的余数作为结果#例:所有人年龄加1显示db.emp.aggregate( {"$project":{ "name":1, "post":1, "new_age":{"$add":["$age",1]} }})# 错误示范: 原因:参加运算的字段不能被影藏db.emp.aggregate( {"$project":{ "name":1, "salary":1, "age":0, "new_age":{"$add":["$age",1]} }}) #3、表达式之日期表达式:$year,$month,$week,$dayOfMonth,$dayOfWeek,$dayOfYear,$hour,$minute,$second#例如：select name,date_format("%Y") as hire_year from db1.empdb.emp.aggregate( {"$project":{"name":1,"hire_year":{"$year":"$hire_date"}}})#例如查看每个员工的工作多长时间
db.emp.aggregate( {"$project":{"name":1,"hire_period":{ "$subtract":[ {"$year":new Date()}, {"$year":"$hire_date"} ] }}})
#4、字符串表达式
{"$substr":[字符串/$值为字符串的字段名,起始位置,截取几个字节]}db.emp.aggregate({"$project":{"new_name":{"$substr":["$name",0,3]}}}){"$concat":[expr1,expr2,...,exprN]} #指定的表达式或字符串连接在一起返回,只支持字符串拼接db.emp.aggregate({"$project":{"new_name":{"$concat":["$name","$post"]}}}){"$toLower":expr}
{"$toUpper":expr}
db.emp.aggregate({"$project":{"new_name":{"$toUpper":"$post"}}})
db.emp.aggregate( {"$project":{"NAME":{"$toUpper":"$name"}}})#5、逻辑表达式
$and$or$not其他见Mongodb权威指南```
## $group
```python
# $group用于分组# 分组后具体信息被影藏 db.emp.aggregate( {"$match":{"_id":{"$gt":3}}}, {"$group":{"_id":"$post"}} )# 通常我们要对分组后的内容进行统计这就需要对应的几个聚合函数
# select id,avg(salary) from db1.emp where id > 3 group by post;
db.emp.aggregate( {"$match":{"_id":{"$gt":3}}}, {"$group":{"_id":"$post",'avg_salary':{"$avg":"$salary"}}},)# math用于匹配 与mysql不同的是没有顺序限制 每一个操作像是一个管道接收上一个的数据进行处理再传给下一个# select id,avg(salary) from db1.emp where id > 3 group by post having avg(salary) > 10000;
db.emp.aggregate( {"$match":{"_id":{"$gt":3}}}, {"$group":{"_id":"$post",'avg_salary':{"$avg":"$salary"}}}, {"$match":{"avg_salary":{"$gt":10000}}}) # 对应的聚合函数 $sum、$avg、$max、$min、$first、$last #1、将分组字段传给$group函数的_id字段即可{"$group":{"_id":"$sex"}} #按照性别分组{"$group":{"_id":"$post"}} #按照职位分组{"$group":{"_id":{"state":"$state","city":"$city"}}} #按照多个字段分组，比如按照州市分组#2、分组后聚合得结果,类似于sql中聚合函数的聚合操作符：$sum、$avg、$max、$min、$first、$last
#例1：select post,max(salary) from db1.emp group by post; db.emp.aggregate({"$group":{"_id":"$post","max_salary":{"$max":"$salary"}}})#例2：去每个部门最大薪资与最低薪资
db.emp.aggregate({"$group":{"_id":"$post","max_salary":{"$max":"$salary"},"min_salary":{"$min":"$salary"}}})#例3：如果字段是排序后的，那么$first,$last会很有用,比用$max和$min效率高
db.emp.aggregate({"$group":{"_id":"$post","first_id":{"$first":"$_id"}}})#例4：求每个部门的总工资
db.emp.aggregate({"$group":{"_id":"$post","count":{"$sum":"$salary"}}})#例5：求每个部门的人数
db.emp.aggregate({"$group":{"_id":"$post","count":{"$sum":1}}}) #3、数组操作符{"$addToSet":expr}：不重复{"$push":expr}：重复# 等同于group_concat#例：查询岗位名以及各岗位内的员工姓名:select post,group_concat(name) from db1.emp group by post;db.emp.aggregate({"$group":{"_id":"$post","names":{"$push":"$name"}}})db.emp.aggregate({"$group":{"_id":"$post","names":{"$addToSet":"$name"}}})```## $sort ,limit,skip
```python
{"$sort":{"字段名":1,"字段名":-1}} #1升序，-1降序{"$limit":n} {"$skip":n} #跳过多少个文档#例1、取平均工资最高的前两个部门db.emp.aggregate(
{ "$group":{"_id":"$post","平均工资":{"$avg":"$salary"}}},{ "$sort":{"平均工资":-1}},{ "$limit":2})#例2、db.emp.aggregate({ "$group":{"_id":"$post","平均工资":{"$avg":"$salary"}}},{ "$sort":{"平均工资":-1}},{ "$limit":2},{ "$skip":1})排序：$sort、限制：$limit、跳过：$skip```## $sample
```python
# 随机取出n条记录#集合users包含的文档如下{ "_id" : 1, "name" : "dave123", "q1" : true, "q2" : true }{ "_id" : 2, "name" : "dave2", "q1" : false, "q2" : false }{ "_id" : 3, "name" : "ahn", "q1" : true, "q2" : true }{ "_id" : 4, "name" : "li", "q1" : true, "q2" : false }{ "_id" : 5, "name" : "annT", "q1" : false, "q2" : true }{ "_id" : 6, "name" : "li", "q1" : true, "q2" : true }{ "_id" : 7, "name" : "ty", "q1" : false, "q2" : true }#下述操作时从users集合中随机选取3个文档
db.users.aggregate({"$sample":{"size":3}})随机选取n个：$sample```
# 可视化工具
https://robomongo.org
from selenium.webdriver import Chrome from urllib.parse import urlencode from selenium.webdriver.common.by import By from selenium.webdriver.support import expected_conditions as EC from selenium.webdriver.support.wait import WebDriverWait import time import mongo driver=Chrome() name='黄金' encode_dict=urlencode({"keyword":name,"enc":"utf-8","wq":name}) url='https://search.jd.com/Search?'+encode_dict driver.get(url) def spider(): WebDriverWait(driver, 10).until(EC.element_to_be_clickable((By.CLASS_NAME, 'pn-next'))) height = driver.execute_script("return document.body.clientHeight") driver.execute_script(""" window.scrollTo({ top: %s, behavior: "smooth" });""" % height) items = driver.find_elements_by_css_selector(".gl-item") if len(items)==60: for i in items: price=i.find_element_by_css_selector('.p-price i').text img=i.find_element_by_css_selector('.p-scroll img').get_attribute('src') url=i.find_element_by_css_selector('.p-img a').get_attribute('href') print(price) mongo_dict={'img':img,'price':price,'url':url} mongo.insert(mongo_dict) WebDriverWait(driver, 30).until(EC.element_to_be_clickable((By.CLASS_NAME, 'pn-next'))) next_page = driver.find_element_by_css_selector('.pn-next') next_page.click() time.sleep(3) spider() else: return spider() for i in range(5): time.sleep(1) spider() time.sleep(6) driver.close() from pymongo import MongoClient import datetime c=MongoClient(host='127.0.0.1',port=27017) db=c['admin'] db.authenticate('root','123') db.c['db1'] def insert(data): c['db1']['jingdong'].insert(data) if __name__=='__main__': insert({"url":"24qew"})