零散知识点

零散知识点

%hive
create external table 库名.表名(
   列名 string（列类型）,
    列名 string（列类型）,
  列名 string（列类型）
)

row format serde 'org.apache.hadoop.hive.serde2.OpenCSVSerde'
with serdeproperties 
(
    'separatorChar' = ',',
    'quoteChar'     = '"',
    'escapeChar'    = '\'
)
location '/目录1/目录2/目录3/'
tblproperties('skip.header.line.count'='1')

create external table spu_db.ex_spu_hbase(
    key string,
    sales double,
    praise int
)
stored by 'org.apache.hadoop.hive.hbase.HbaseStorageHandler'
with serdeproperties(
"hbase.columns.mapping"=":key,result:sales,result:praise"
)
tblproperties(
"hbase.table.name"="exam:spu"
)

%hive
create external table 库名.表名(
    列名 string（列类型）,
    列名 string（列类型）,
  列名 string（列类型）
)
row format delimited fields terminated by ','
location '/目录1/目录2/目录3/'
tblproperties('skip.header.line.count'='1')

一个目录里只能放一个文件

%hive
insert overwrite spu_db.ex_spu_hbase
select concat(shop_id,shop_name) key,sum(month_sales*spu_price) sales,count(praise_num) from spu_db.ex_spu group by shop_id,shop_name

%spark
val rdd=sc.textFile("hdfs://192.168.126.200:9000/目录1/目录2/文件名.csv")
val head = rdd.first()
rdd.filter(_!head).count()

-- 方法2
val df = spark.read.format("csv").option("header","true").load("hdfs://192.168.126.200:9000/目录1/目录2/文件名.csv")
df.createOrReplaceTempView("sales")

-- 方法3
spark.sql("""select count(distinct orderid) from sales""").show()

%hive
insert into myhbase.hbase_userinfos
select concat(month,quetype) key, count(distinct orderid) serviceReasonDetailCount from exam.ex_exam_after_sales_service group by month,quetype