Audio Recording (#11887)

* Audio capture

* Better filename

* Convert to MP3

* Add override for mime types (#11892)

* Add override.

* Better naming

* Better label

* Use theme var

* Fix params.

Co-authored-by: Marcin Pietruszka <marcin@webskill.pl>

* Move vars

* Nicer for the eye

* Karma test

* Record Video/Audio

Co-authored-by: Pascal Birchler <pascalb@google.com>

* Pick keys from resource for background audio

* Unify icon attrs

* Fix tests

* Fix tests

Co-authored-by: Jonny Harris <spacedmonkey@users.noreply.github.com>
Co-authored-by: Pascal Birchler <pascalb@google.com>

packages/design-system/src/icons/index.js

@@ -48,6 +48,7 @@ export { default as Box4 } from './box4.svg';
 export { default as Box4Alternate } from './box4_alternate.svg';
 export { default as Bucket } from './bucket.svg';
 export { default as Camera } from './camera.svg';
+export { default as CameraOff } from './camera_off.svg';
 export { default as Captions } from './captions.svg';
 export { default as Checkbox } from './checkbox.svg';
 export { default as Checklist } from './checklist.svg';

packages/elements/src/types.js

@@ -29,14 +29,17 @@ import { MULTIPLE_VALUE, BACKGROUND_TEXT_MODE, OverlayType } from './constants';
 
 const StoryPropTypes = {};
 
-export const BackgroundAudioPropType = PropTypes.shape({
+export const BackgroundAudioPropTypeShape = {
   id: PropTypes.number,
   src: PropTypes.string,
   length: PropTypes.number,
   lengthFormatted: PropTypes.string,
   mimeType: PropTypes.string,
   needsProxy: PropTypes.bool,
-});
+};
+export const BackgroundAudioPropType = PropTypes.shape(
+  BackgroundAudioPropTypeShape
+);
 
 StoryPropTypes.mask = PropTypes.shape({
   type: PropTypes.string.isRequired,

packages/story-editor/src/app/highlights/quickActions/useQuickActions.js

@@ -72,20 +72,15 @@ const {
   Settings,
 } = Icons;
 
-const StyledSettings = styled(Settings).attrs({
+const quickActionIconAttrs = {
   width: 24,
   height: 24,
-})``;
-
-const Mic = styled(Icons.Mic).attrs({
-  width: 24,
-  height: 24,
-})``;
-
-const MicOff = styled(Icons.MicOff).attrs({
-  width: 24,
-  height: 24,
-})``;
+};
+const StyledSettings = styled(Settings).attrs(quickActionIconAttrs)``;
+const Mic = styled(Icons.Mic).attrs(quickActionIconAttrs)``;
+const MicOff = styled(Icons.MicOff).attrs(quickActionIconAttrs)``;
+const Video = styled(Icons.Camera).attrs(quickActionIconAttrs)``;
+const VideoOff = styled(Icons.CameraOff).attrs(quickActionIconAttrs)``;
 
 export const MediaPicker = ({ render, ...props }) => {
   const {
@@ -302,20 +297,26 @@ const useQuickActions = () => {
   const {
     isInRecordingMode,
     toggleRecordingMode,
+    toggleVideo,
     toggleAudio,
+    hasVideo,
     hasAudio,
     toggleSettings,
     audioInput,
+    videoInput,
     isReady,
   } = useMediaRecording(({ state, actions }) => ({
     isInRecordingMode: state.isInRecordingMode,
     hasAudio: state.hasAudio,
+    hasVideo: state.hasVideo,
     audioInput: state.audioInput,
+    videoInput: state.videoInput,
     isReady:
       state.status === 'ready' &&
       !state.file?.type?.startsWith('image') &&
       !state.isCountingDown,
     toggleRecordingMode: actions.toggleRecordingMode,
+    toggleVideo: actions.toggleVideo,
     toggleAudio: actions.toggleAudio,
     toggleSettings: actions.toggleSettings,
     muteAudio: actions.muteAudio,
@@ -813,15 +814,32 @@ const useQuickActions = () => {
           });
           toggleAudio();
         },
-        disabled: !isReady,
+        disabled: !isReady || !hasVideo,
+        ...actionMenuProps,
+      },
+      videoInput && {
+        Icon: hasVideo ? Video : VideoOff,
+        label: hasVideo
+          ? __('Disable Video', 'web-stories')
+          : __('Enable Video', 'web-stories'),
+        onClick: () => {
+          trackEvent('media_recording_video_toggled', {
+            status: hasVideo ? 'off' : 'on',
+          });
+          toggleVideo();
+        },
+        disabled: !isReady || !hasAudio,
         ...actionMenuProps,
       },
     ].filter(Boolean);
   }, [
     actionMenuProps,
     audioInput,
+    videoInput,
+    hasVideo,
     hasAudio,
     toggleAudio,
+    toggleVideo,
     toggleRecordingMode,
     toggleSettings,
     isReady,

packages/story-editor/src/app/highlights/states.js

@@ -58,6 +58,9 @@ const keys = {
   MEDIA3P: 'MEDIA3P',
   TEXT_SET: 'TEXT',
   PAGE_TEMPLATES: 'PAGE_TEMPLATES',
+
+  // DOCUMENT
+  BACKGROUND_AUDIO: 'BACKGROUND_AUDIO',
 };
 
 export const STATES = {
@@ -84,6 +87,10 @@ export const STATES = {
     focus: true,
     tab: DOCUMENT,
   },
+  [keys.BACKGROUND_AUDIO]: {
+    focus: true,
+    tab: DOCUMENT,
+  },
   [keys.CAPTIONS]: {
     focus: true,
     tab: STYLE,

packages/story-editor/src/app/media/useUploadMedia.js

@@ -278,7 +278,11 @@ function useUploadMedia({
           const isTooLarge = canTranscode && isFileTooLarge(file);
 
           try {
-            validateFileForUpload(file, canTranscode, isTooLarge);
+            validateFileForUpload({
+              file,
+              canTranscodeFile: canTranscode,
+              isFileTooLarge: isTooLarge,
+            });
           } catch (e) {
             showSnackbar({
               message: e.message,

packages/story-editor/src/app/media/utils/useFFmpeg.js

@@ -457,6 +457,59 @@ function useFFmpeg() {
     [getFFmpegInstance]
   );
 
+  /**
+   * Converts any audio file format to MP3 using FFmpeg.
+   *
+   * @param {File} file Original audio file object.
+   * @return {Promise<File>} Converted video file object.
+   */
+  const convertToMp3 = useCallback(
+    async (file) => {
+      //eslint-disable-next-line @wordpress/no-unused-vars-before-return -- False positive because of the finally().
+      const trackTiming = getTimeTracker('load_mp3_conversion');
+
+      let ffmpeg;
+
+      try {
+        ffmpeg = await getFFmpegInstance(file);
+
+        const tempFileName = uuidv4() + '.mp3';
+        const outputFileName = getFileName(file) + '.mp3';
+
+        await ffmpeg.run(
+          // Input filename.
+          '-i',
+          file.name,
+          // Output filename. MUST be different from input filename.
+          tempFileName
+        );
+
+        const data = ffmpeg.FS('readFile', tempFileName);
+
+        return new blobToFile(
+          new Blob([data.buffer], { type: 'audio/mpeg' }),
+          outputFileName,
+          'audio/mpeg'
+        );
+      } catch (err) {
+        // eslint-disable-next-line no-console -- We want to surface this error.
+        console.error(err);
+
+        trackError('mp3_conversion', err.message);
+
+        throw err;
+      } finally {
+        try {
+          ffmpeg.exit();
+          // eslint-disable-next-line no-empty -- no-op
+        } catch (e) {}
+
+        trackTiming();
+      }
+    },
+    [getFFmpegInstance]
+  );
+
   /**
    * Determines whether the given file can be transcoded.
    *
@@ -496,6 +549,7 @@ function useFFmpeg() {
       stripAudioFromVideo,
       getFirstFrameOfVideo,
       convertGifToVideo,
+      convertToMp3,
       trimVideo,
     }),
     [
@@ -505,6 +559,7 @@ function useFFmpeg() {
       stripAudioFromVideo,
       getFirstFrameOfVideo,
       convertGifToVideo,
+      convertToMp3,
       trimVideo,
     ]
   );

packages/story-editor/src/app/uploader/test/useUploader.js

@@ -88,7 +88,7 @@ describe('useUploader', () => {
         },
       });
 
-      await expect(() => validateFileForUpload({})).toThrow(
+      await expect(() => validateFileForUpload({ file: {} })).toThrow(
         'Sorry, you are not allowed to upload files.'
       );
     });
@@ -100,7 +100,9 @@ describe('useUploader', () => {
         maxUpload: 2000000,
       });
 
-      await expect(() => validateFileForUpload({ size: 3000000 })).toThrow(
+      await expect(() =>
+        validateFileForUpload({ file: { size: 3000000 } })
+      ).toThrow(
         'Your file is 3MB and the upload limit is 2MB. Please resize and try again!'
       );
     });
@@ -111,23 +113,38 @@ describe('useUploader', () => {
       } = setup({});
 
       await expect(() =>
-        validateFileForUpload({ size: 20000, type: 'video/quicktime' })
+        validateFileForUpload({
+          file: { size: 20000, type: 'video/quicktime' },
+        })
       ).toThrow(
         'Please choose only png, jpg, jpeg, gif, webp, mp4, or webm to upload.'
       );
     });
 
+    it('throws an error if file type is not supported and in list of mime types', async () => {
+      const {
+        actions: { validateFileForUpload },
+      } = setup({});
+
+      await expect(() =>
+        validateFileForUpload({
+          file: { size: 20000, type: 'video/quicktime' },
+          overrideAllowedMimeTypes: ['video/mp4'],
+        })
+      ).toThrow('Please choose only mp4 to upload.');
+    });
+
     it('throws an error if file too large to transcode', async () => {
       const {
         actions: { validateFileForUpload },
       } = setup({});
 
       await expect(() =>
-        validateFileForUpload(
-          { size: 1024 * 1024 * 1024 * 2, type: 'video/mp4' },
-          true,
-          true
-        )
+        validateFileForUpload({
+          file: { size: 1024 * 1024 * 1024 * 2, type: 'video/mp4' },
+          canTranscodeFile: true,
+          isFileTooLarge: true,
+        })
       ).toThrow(
         'Your file is too large (2048 MB) and cannot be processed. Please try again with a file that is smaller than 2048 MB.'
       );
@@ -141,14 +158,11 @@ describe('useUploader', () => {
       });
 
       await expect(() =>
-        validateFileForUpload(
-          {
-            size: 1024 * 1024 * 1024 * 3,
-            type: 'video/quicktime',
-          },
-          true,
-          true
-        )
+        validateFileForUpload({
+          file: { size: 1024 * 1024 * 1024 * 3, type: 'video/quicktime' },
+          canTranscodeFile: true,
+          isFileTooLarge: true,
+        })
       ).toThrow(
         'Your file is too large (3072 MB) and cannot be processed. Please try again with a file that is smaller than 2048 MB.'
       );
@@ -160,11 +174,11 @@ describe('useUploader', () => {
       } = setup({});
 
       await expect(() =>
-        validateFileForUpload(
-          { size: 1024 * 1024 * 150, type: 'video/mp4' },
-          true,
-          false
-        )
+        validateFileForUpload({
+          file: { size: 1024 * 1024 * 150, type: 'video/mp4' },
+          canTranscodeFile: true,
+          isFileTooLarge: false,
+        })
       ).not.toThrow();
     });
 
@@ -174,11 +188,11 @@ describe('useUploader', () => {
       } = setup({});
 
       await expect(() =>
-        validateFileForUpload(
-          { size: 1024 * 1024 * 1024 * 2, type: 'video/mp4' },
-          false,
-          true
-        )
+        validateFileForUpload({
+          file: { size: 1024 * 1024 * 1024 * 2, type: 'video/mp4' },
+          canTranscodeFile: false,
+          isFileTooLarge: true,
+        })
       ).toThrow(
         'Your file is 2048MB and the upload limit is 100MB. Please resize and try again!'
       );
@@ -190,11 +204,11 @@ describe('useUploader', () => {
       } = setup({});
 
       await expect(() =>
-        validateFileForUpload(
-          { size: 1024 * 1024 * 50, type: 'video/mp4' },
-          false,
-          false
-        )
+        validateFileForUpload({
+          file: { size: 1024 * 1024 * 50, type: 'video/mp4' },
+          canTranscodeFile: false,
+          isFileTooLarge: false,
+        })
       ).not.toThrow();
     });
 
@@ -206,7 +220,9 @@ describe('useUploader', () => {
       });
 
       await expect(() =>
-        validateFileForUpload({ size: 20000, type: 'video/quicktime' })
+        validateFileForUpload({
+          file: { size: 20000, type: 'video/quicktime' },
+        })
       ).toThrow('Please choose only mp4 to upload.');
     });
 
@@ -216,7 +232,9 @@ describe('useUploader', () => {
       } = setup({ allowedMimeTypes: { image: [], video: [], vector: [] } });
 
       await expect(() =>
-        validateFileForUpload({ size: 20000, type: 'video/quicktime' })
+        validateFileForUpload({
+          file: { size: 20000, type: 'video/quicktime' },
+        })
       ).toThrow('No file types are currently supported.');
     });
   });