tencent cloud

PrevNext

Feedback

search by keyword

Recent Pages

Documentation
Chat
    Documentation
    Download PDF

    Flutter

    Last updated: 2024-03-20 16:26:08
    Download PDF
      This document mainly introduces how to integrate beauty effects in TUICallKit.
      To perform custom beauty processing in Flutter, it is necessary to use TRTC's custom video rendering. Due to Flutter's lack of proficiency in handling real-time transmission of large data volumes, work involving TRTC's custom video rendering needs to be completed in the Native section. The specific plan is as follows:
      
      The integration plan is divided into 3 steps:
      Step 1: Enable/Disable the TRTC custom rendering logic through MethodChannel; Step 2: In the TRTC custom rendering processing logic onProcessVideoFrame(), use the beauty processing module to process the original video frames; Step 3: On the user's beauty processing module, the beauty parameters also need to be set through the Dart interface. Users can set beauty parameters through MethodChannel based on their needs and the beauty effects they use.

      Integrating Third-Party Beauty Effects

      Step 1: Implement the control interface of enabling/disabling beauty effects from Dart to Native.

      Implement the interface in Dart:
       final channel = MethodChannel('TUICallKitCustomBeauty');
        
       void enableTUICallKitCustomBeauty() async {
          await channel.invokeMethod('enableTUICallKitCustomBeauty');   
       }
      
       void disableTUICallKitCustomBeauty() async {
          await channel.invokeMethod('disableTUICallKitCustomBeauty');
       }
      Implement the corresponding interface in Native:
      Java
      Swift
      public class MainActivity extends FlutterActivity {
          private static final String channelName = "TUICallKitCustomBeauty";
          
          private MethodChannel channel;
          
          @Override
          public void configureFlutterEngine(@NonNull FlutterEngine flutterEngine) {
              super.configureFlutterEngine(flutterEngine);
              
              channel = new MethodChannel(flutterEngine.getDartExecutor().getBinaryMessenger(), channelName);
              channel.setMethodCallHandler(((call, result) -> {
                  switch (call.method) {
                      case "enableTUICallKitCustomBeauty":
                          enableTUICallKitCustomBeauty();
                          break;
                      case "disableTUICallKitCustomBeauty":
                          disableTUICallKitCustomBeauty();
                          break;
                      default:
                          break;
                  }
                  result.success("");
              }));
          }
          
          public void enableTUICallKitCustomBeauty() {
          
          }
          
          public void disableTUICallKitCustomBeauty() {
          
          }
      }J
      @UIApplicationMain
      @objc class AppDelegate: FlutterAppDelegate {
          var channel: FlutterMethodChannel?
          
          override func application(_ application: UIApplication,
                                    didFinishLaunchingWithOptions launchOptions: [UIApplication.LaunchOptionsKey: Any]?) -> Bool {
              GeneratedPluginRegistrant.register(with: self)
         
              guard let controller = window?.rootViewController as? FlutterViewController else {
                  fatalError("Invalid root view controller")
              }
              channel = FlutterMethodChannel(name: "TUICallKitCustomBeauty", binaryMessenger: controller.binaryMessenger)
              channel?.setMethodCallHandler({ [weak self] call, result in
                  guard let self = self else { return }
                  switch (call.method) {
                  case "enableTUICallKitCustomBeauty":
                      self.enableTUICallKitCustomBeauty()
                      break
                  case "disableTUICallKitCustomBeauty":
                      self.disableTUICallKitCustomBeauty()
                      break
                  default:
                      break
                  }
              })    
              result(nil)   
              return super.application(application, didFinishLaunchingWithOptions: launchOptions)
          }
          
          func enableTUICallKitCustomBeauty() {
          
          }
          
          func disableTUICallKitCustomBeauty() {
          
          }
      }S

      Step 2: Complete the beauty processing in the TRTC custom rendering logic in Native.

      Note:
      Android requires a dependency on LiteAVSDK_Professional during the beauty integration process. Add the following dependency in the Android project's app/build.gradle: dependencies{ api "com.tencent.liteav:LiteAVSDK_Professional:latest.release" }
      Java
      Swift
      Jvoid enableTUICallKitCustomBeauty() {
          TRTCCloud.sharedInstance(getApplicationContext()).
                      setLocalVideoProcessListener(TRTC_VIDEO_PIXEL_FORMAT_Texture_2D,
                              TRTC_VIDEO_BUFFER_TYPE_TEXTURE, new VideoFrameListerer());
      
      }
      
      void disableTUICallKitCustomBeauty() {
          TRTCCloud.sharedInstance(getApplicationContext()).
                      setLocalVideoProcessListener(TRTC_VIDEO_PIXEL_FORMAT_Texture_2D,
                              TRTC_VIDEO_BUFFER_TYPE_TEXTURE, null);
      }
      
      class VideoFrameListerer implements TRTCCloudListener.TRTCVideoFrameListener {
    private XXXBeautyModel  mBeautyModel = XXXBeautyModel.sharedInstance(); 
         
    @Override
    public int onProcessVideoFrame(TRTCCloudDef.TRTCVideoFrame trtcVideoFrame,
                                   TRTCCloudDef.TRTCVideoFrame trtcVideoFrame1) {
        //Beauty processing logic
              mBeautyModel.process(trtcVideoFrame, trtcVideoFrame1);
              …
              
        return 0;
    }

    @Override
    public void onGLContextCreated() {
    }
    
    @Override
    public void onGLContextDestory() {
    }
}
      let videoFrameListener: TRTCVideoFrameListener = TRTCVideoFrameListener()
      
      func enableTUICallKitCustomBeauty() {
          TRTCCloud.sharedInstance().setLocalVideoProcessDelegete(videoFrameListener, pixelFormat: ._Texture_2D, bufferType: .texture)
      }
      
      func disableTUICallKitCustomBeauty() {
          TRTCCloud.sharedInstance().setLocalVideoProcessDelegete(nil, pixelFormat: ._Texture_2D, bufferType: .texture)
      }
      
      class TRTCVideoFrameListener: NSObject, TRTCVideoFrameDelegate {
          let bueutyModel = XXXXBeautyModel.shareIntance()
          func onProcessVideoFrame(_ srcFrame: TRTCVideoFrame, dstFrame: TRTCVideoFrame) -> UInt32 {
              //Beauty processing logic
              bueutyModel.onProcessVideoFrame(srcFrame, dstFrame)
              …
              
              return 0
          }
      }S

      Step 3: Customize third-party beauty parameter control logic.

      In this step, users can use a specific beauty module as needed. See Step 1 for the implementation of beauty parameter settings. The specific implementation depends on the actual use case.

      Integrating Tencent Beauty Effects

      The method of integrating Tencent beauty effects also follows the above-mentioned steps. The following description takes Tencent beauty effects as an example to illustrate the method of integration.

      Step 1: Download and integrate beauty effect resources.

      1. Based on the package you purchased, Download SDK.
      2. Add the resource files to your own project:
      Android
      iOS
      1. In the app module, find the build.gradle file and add the Maven reference address corresponding to your package. For example, if you have chosen the S1-04 package, add the following code:
      dependencies {
         implementation 'com.tencent.mediacloud:TencentEffect_S1-04:latest.release'
      }
      For the Maven address corresponding to each package, refer to Documentation.
      2. In the app module, find the src/main/assets folder, or create one if it does not exist. Check if the downloaded SDK package includes the MotionRes folder, and if so, copy this folder to the ../src/main/assets directory.
      3. Find the AndroidManifest.xml file in the app module, and add the following tag in the application element.
       <uses-native-library
              android:name="libOpenCL.so"
              android:required="true" />
              //The "true" here indicates that if the library is unavailable, the application will not be able to run normally. The system does not allow installation of the application on devices without this library.
              //The "false" indicates that the application can utilize the library (if available), but is specifically designed to run without the library (if necessary). The system allows the application to be installed, even in the absence of this library. If you use "false", you need to properly handle the absence of the library.
              //Official Android website introduction: %!s(<nil>)
      The tag is added, as shown in the following figure:
      
      4. Aliasing Configuration
      If you enable compile optimizations while building the release package (by setting minifyEnabled to true), some codes that are not called at the Java level might be trimmed. However, these codes could potentially be called at the Native level, leading to a no xxx method exception.
      If you enable such compile optimizations, you must add these "keep" rules to prevent the codes of xmagic from being trimmed:
      -keep class com.tencent.xmagic.** { *;}
      -keep class org.light.** { *;}
      -keep class org.libpag.** { *;}
      -keep class org.extra.** { *;}
      -keep class com.gyailib.**{ *;}
      -keep class com.tencent.cloud.iai.lib.** { *;}
      -keep class com.tencent.beacon.** { *;}
      -keep class com.tencent.qimei.** { *;}
      -keep class androidx.exifinterface.** { *;}
      1. Add beauty effect resources to your project, as shown in the following figure (your resources may not exactly match those in the following figure):
      
      2. In the demo, copy the following four classes from demo/lib/producer to your own Flutter project: BeautyDataManager, BeautyPropertyProducer, BeautyPropertyProducerAndroid, and BeautyPropertyProducerIOS. These four classes are used to configure beauty effect resources and display the beauty types on the beauty panel.

      Step 2: Reference the Flutter SDK.

      GitHub Reference: Add the following reference in your project's pubspec.yaml file:
       tencent_effect_flutter:
         git:
           url: https://github.com/TencentCloud/tencenteffect-sdk-flutter
      Local Reference: Download the latest version of tencent_effect_flutter, then add the folders android, ios, lib, and the files pubspec.yaml, tencent_effect_flutter.iml to the project directory. Next, add the following reference in your project's pubspec.yaml file (you can refer to the demo) :
      tencent_effect_flutter:
          path: ../
      tencent_effect_flutter merely provides a bridge, and it is the XMagic that is has dependancy on, which is set to the latest version by default, that actually implements beauty effects.
      To use the latest version of the beauty SDK, you can upgrade the SDK by following these steps:
      Android
      iOS
      Execute the flutter pub upgrade command in the project directory or click Pub upgrade in the top right corner of the subspec.yaml page.
      Execute the flutter pub upgrade command in the project directory, and then execute the pod update command in the iOS directory.

      Step 3: Implement the control interface of enabling/disabling beauty effects from Dart to Native.

      Step 4: Complete the beauty processing in the TRTC custom rendering logic in Native.

      Android
      iOS
      Android requires a dependency on LiteAVSDK_Professional during the beauty integration process. Add the following dependency in the Android project's app/build.gradle: dependencies{ api "com.tencent.liteav:LiteAVSDK_Professional:latest.release" }
      void enableTUICallKitCustomBeauty() {
          TRTCCloud.sharedInstance(getApplicationContext()).
                      setLocalVideoProcessListener(TRTC_VIDEO_PIXEL_FORMAT_Texture_2D,
                              TRTC_VIDEO_BUFFER_TYPE_TEXTURE, new VideoFrameListerer());
      
      }
      
      void disableTUICallKitCustomBeauty() {
          TRTCCloud.sharedInstance(getApplicationContext()).
                      setLocalVideoProcessListener(TRTC_VIDEO_PIXEL_FORMAT_Texture_2D,
                              TRTC_VIDEO_BUFFER_TYPE_TEXTURE, null);
      }
      
      class VideoFrameListerer implements TRTCCloudListener.TRTCVideoFrameListener {
          private XXXBeautyModel  mBeautyModel = XXXBeautyModel.sharedInstance(); 
         
          @Override
          public int onProcessVideoFrame(TRTCCloudDef.TRTCVideoFrame trtcVideoFrame,
                                         TRTCCloudDef.TRTCVideoFrame trtcVideoFrame1) {
             trtcVideoFrame1.texture.textureId = XmagicApiManager.getInstance()
        .process(trtcVideoFrame.texture.textureId, trtcVideoFrame.width, trtcVideoFrame.height);
              return 0;
          }
      
          @Override
          public void onGLContextCreated() {
          }
      
          @Override
          public void onGLContextDestory() {
          }
      }
      import TXLiteAVSDK_Professional
      import tencent_effect_flutter
      
      let videoFrameListener: TRTCVideoFrameListener = TRTCVideoFrameListener()
      
      func enableTUICallKitCustomBeauty() {
          TRTCCloud.sharedInstance().setLocalVideoProcessDelegete(videoFrameListener, pixelFormat: ._Texture_2D, bufferType: .texture)
      }
      
      func disableTUICallKitCustomBeauty() {
          TRTCCloud.sharedInstance().setLocalVideoProcessDelegete(nil, pixelFormat: ._Texture_2D, bufferType: .texture)
      }
      
      class TRTCVideoFrameListener: NSObject, TRTCVideoFrameDelegate {
          func onProcessVideoFrame(_ srcFrame: TRTCVideoFrame, dstFrame: TRTCVideoFrame) -> UInt32 {
              dstFrame.textureId = GLuint(XmagicApiManager.shareSingleton().getTextureId(ConvertBeautyFrame.convertTRTCVideoFrame(trtcVideoFrame: srcFrame)))
              return 0
          }
      }
      
      
      public class ConvertBeautyFrame: NSObject {
          public static func convertToTRTCPixelFormat(beautyPixelFormat: ITXCustomBeautyPixelFormat) -> TRTCVideoPixelFormat {
              switch beautyPixelFormat {
              case .Unknown:
                  return ._Unknown
              case .I420:
                  return ._I420
              case .Texture2D:
                  return ._Texture_2D
              case .BGRA:
                  return ._32BGRA
              case .NV12:
                  return ._NV12
              }
          }
      
          public static func convertTRTCVideoFrame(trtcVideoFrame: TRTCVideoFrame) -> ITXCustomBeautyVideoFrame {
              let beautyVideoFrame = ITXCustomBeautyVideoFrame()
              beautyVideoFrame.data = trtcVideoFrame.data
              beautyVideoFrame.pixelBuffer = trtcVideoFrame.pixelBuffer
              beautyVideoFrame.width = UInt(trtcVideoFrame.width)
              beautyVideoFrame.height = UInt(trtcVideoFrame.height)
              beautyVideoFrame.textureId = trtcVideoFrame.textureId
              switch trtcVideoFrame.rotation {
              case ._0:
                  beautyVideoFrame.rotation = .rotation_0
              case ._90:
                  beautyVideoFrame.rotation = .rotation_90
              case ._180:
                  beautyVideoFrame.rotation = .rotation_180
              case ._270:
                  beautyVideoFrame.rotation = .rotation_270
              default:
                  beautyVideoFrame.rotation = .rotation_0
              }
              switch trtcVideoFrame.pixelFormat {
              case ._Unknown:
                  beautyVideoFrame.pixelFormat = .Unknown
              case ._I420:
                  beautyVideoFrame.pixelFormat = .I420
              case ._Texture_2D:
                  beautyVideoFrame.pixelFormat = .Texture2D
              case ._32BGRA:
                  beautyVideoFrame.pixelFormat = .BGRA
              case ._NV12:
                  beautyVideoFrame.pixelFormat = .NV12
              default:
                  beautyVideoFrame.pixelFormat = .Unknown
              }
              beautyVideoFrame.bufferType = ITXCustomBeautyBufferType(rawValue: trtcVideoFrame.bufferType.rawValue) ?? .Unknown
              beautyVideoFrame.timestamp = trtcVideoFrame.timestamp
              return beautyVideoFrame
          }
      }

      Step 5: Enable beauty effects and set beauty parameters.

      After completing the configurations above, you can enable/disable beauty effects by using enableTUICallKitCustomBeauty()/disableTUICallKitCustomBeauty() . You can set beauty parameters through the Tencent Effect beauty Flutter interface.
      
      Contact Us

      Contact our sales team or business advisors to help your business.

      Contact Us
      Technical Support

      Open a ticket if you're looking for further assistance. Our Ticket is 7x24 avaliable.

      Submit a Ticket
      7x24 Phone Support
      Copyright © 2013-2024 Tencent Cloud. All Rights Reserved.
      Privacy PolicyLegalCookie Policy