tencent cloud

<dependency>
  <groupId>org.apache.spark</groupId>
  <artifactId>spark-core_2.12</artifactId>
  <version>3.2.1</version>
  <scope>provided</scope>
</dependency>
<dependency>
  <groupId>org.apache.spark</groupId>
  <artifactId>spark-sql_2.12</artifactId>
  <version>3.2.1</version>
  <scope>provided</scope>
</dependency>

package com.tencent.dlc;

import org.apache.spark.sql.Dataset;
import org.apache.spark.sql.Row;
import org.apache.spark.sql.SaveMode;
import org.apache.spark.sql.SparkSession;


public class CosService {

    public static void main( String[] args )
    {
        //1. Create SparkSession
        SparkSession spark = SparkSession
                .builder()
                .appName("Operate data on cos")
                .config("spark.some.config.option", "some-value")
                .getOrCreate();
        //2. Read the json file from COS to generate a data set, supporting various file types such as json, csv, parquet, orc, text
        String readPath = "cosn://dlc-demo-1305424723/people.json";
        Dataset<Row> readData = spark.read().json(readPath);
        //3. Perform business computations on the data set to generate result data, supporting API and SQL formats. Here, a temporary table is generated and SQL is used to read data.
        readData.createOrReplaceTempView("people");
        Dataset<Row> result = spark.sql("SELECT * FROM people where age > 3");
        //4. Save the result data back to COS
         String writePath = "cosn://dlc-demo-1305424723/people_output";
        //Supports writing various file types such as json, csv, parquet, orc, text
        result.write().mode(SaveMode.Append).json(writePath);
        spark.read().json(writePath).show();
        //5. Close the session
        spark.stop();
    }
}

package com.tencent.dlc;

import org.apache.spark.sql.SparkSession;


public class DbService {

    public static void main(String[] args) {
        //1. Initialize SparkSession
        SparkSession spark = SparkSession
                .builder()
                .appName("Operate DB Example")
                .getOrCreate();
        //2. Create a database
        String dbName = " `DataLakeCatalog`.`dlc_db_test` ";
        String dbSql = "CREATE DATABASE IF NOT EXISTS" + dbName + " COMMENT 'demo test'";
        spark.sql(dbSql);
        //3. Create an internal table
        String tableName = "`test`";
        String tableSql = "CREATE TABLE IF NOT EXISTS " + dbName + "." + tableName
                + "(`id` int,`name` string, `age` int)";
        spark.sql(tableSql);
        //4. Write data
        spark.sql("INSERT INTO " + dbName + "." + tableName + "VALUES (1,'Andy',12),(2,'Justin',3) ");
        //5. Query data
        spark.sql(" SELECT * FROM " + dbName + "." + tableName).show();
        //6. Create an external table
        String extTableName = "`ext_test`";
        spark.sql(
                "CREATE EXTERNAL TABLE IF NOT EXISTS " + dbName + "." + extTableName + ""
                        + "  (`id` int, `name` string, `age` int) "
                        + "ROW FORMAT SERDE 'org.apache.hive.hcatalog.data.JsonSerDe' "
                        + "STORED AS TEXTFILE LOCATION 'cosn://dlc-demo-1305424723/ext_test '");
        //7. Write data to the external table
        spark.sql("INSERT INTO " + dbName + "." + extTableName + "VALUES (1,'LiLy',12),(2,'Lucy',3) ");
        //8. Query data from the external table
        spark.sql(" SELECT * FROM " + dbName + "." + extTableName).show();
        //9. Close the session
        spark.stop();
    }
}