closeTab() from lightning:workspaceAPI can be used to end the Messaging Session.

1. Create the following Lightning Aura Component.

Component:

<aura:component implements="flexipage:availableForAllPageTypes" access="global">
    <lightning:workspaceAPI aura:id="workspace"/>
    <lightning:button label="End Messaging Session" onclick="{!c.closeFocusedTab}"/>
</aura:component> 

JavaScript:

( {
    
    closeFocusedTab : function( component, event, helper ) {
        
        let workspaceAPI = component.find( "workspace" );
        workspaceAPI.getFocusedTabInfo().then( function( response ) {
            let focusedTabId = response.tabId;
            workspaceAPI.closeTab( { tabId: focusedTabId } );
        })
        .catch( function( error ) {
            console.log( JSON.stringify( error ) );
        });
        
    }
    
} )

2. Add the Lightning Aura Component to the Messaging Session Lightning Record Page.

Output:

When you click the End Messaging Session button, it was show the Prompt to end the conversation. Click End Conversation to end the Messaging Session.