tencent cloud

<dependency>
  <groupId>org.apache.spark</groupId>
  <artifactId>spark-core_2.12</artifactId>
  <version>3.2.1</version>
  <scope>provided</scope>
</dependency>
<dependency>
  <groupId>org.apache.spark</groupId>
  <artifactId>spark-sql_2.12</artifactId>
  <version>3.2.1</version>
  <scope>provided</scope>
</dependency>

package com.tencent.dlc;

import org.apache.spark.sql.Dataset;
import org.apache.spark.sql.Row;
import org.apache.spark.sql.SaveMode;
import org.apache.spark.sql.SparkSession;


public class CosService {

    public static void main( String[] args )
    {
        //1.创建SparkSession
        SparkSession spark = SparkSession
                .builder()
                .appName("Operate data on cos")
                .config("spark.some.config.option", "some-value")
                .getOrCreate();
        //2.读取cos上的json文件生成数据集,支持多种类型的文件，如 json,csv,parquet,orc,text
        String readPath = "cosn://dlc-demo-1305424723/people.json";
        Dataset<Row> readData = spark.read().json(readPath);
        //3.对数据集做业务计算操作生成结果数据，计算支持API和SQL形式，这里生成临时表用sql读数据
        readData.createOrReplaceTempView("people");
        Dataset<Row> result = spark.sql("SELECT * FROM people where age > 3");
        //4.结果数据保存到cos
         String writePath = "cosn://dlc-demo-1305424723/people_output";
        //写入支持多种类型的文件，如 json,csv,parquet,orc,text
        result.write().mode(SaveMode.Append).json(writePath);
        spark.read().json(writePath).show();
        //5.关闭session
        spark.stop();
    }
}

package com.tencent.dlc;

import org.apache.spark.sql.SparkSession;


public class DbService {

    public static void main(String[] args) {
        //1.初始化SparkSession
        SparkSession spark = SparkSession
                .builder()
                .appName("Operate DB Example")
                .getOrCreate();
        //2.建数据库
        String dbName = " `DataLakeCatalog`.`dlc_db_test` ";
        String dbSql = "CREATE DATABASE IF NOT EXISTS" + dbName + " COMMENT 'demo test'";
        spark.sql(dbSql);
        //3.建内表
        String tableName = "`test`";
        String tableSql = "CREATE TABLE IF NOT EXISTS " + dbName + "." + tableName
                + "(`id` int,`name` string, `age` int)";
        spark.sql(tableSql);
        //4.写数据
        spark.sql("INSERT INTO " + dbName + "." + tableName + "VALUES (1,'Andy',12),(2,'Justin',3) ");
        //5.查询数据
        spark.sql(" SELECT * FROM " + dbName + "." + tableName).show();
        //6.建外表
        String extTableName = "`ext_test`";
        spark.sql(
                "CREATE EXTERNAL TABLE IF NOT EXISTS " + dbName + "." + extTableName + ""
                        + "  (`id` int, `name` string, `age` int) "
                        + "ROW FORMAT SERDE 'org.apache.hive.hcatalog.data.JsonSerDe' "
                        + "STORED AS TEXTFILE LOCATION 'cosn://dlc-demo-1305424723/ext_test '");
        //7.写外表数据
        spark.sql("INSERT INTO " + dbName + "." + extTableName + "VALUES (1,'LiLy',12),(2,'Lucy',3) ");
        //8.查询外表数据
        spark.sql(" SELECT * FROM " + dbName + "." + extTableName).show();
        //9.关闭Session
        spark.stop();
    }
}