For openSubtab and openTab in Salesforce Lightning Web Component, we have to use lightning/platformWorkspaceApi.

Sample Lightning Web Component:

HTML:

<template>
    <lightning-card>
        <div class="slds-var-m-around_medium">
            <lightning-datatable
                key-field="Id"
                data={records}
                columns={columns}
                hide-checkbox-column
                show-row-number-column
                onrowaction={handleRowAction}>
            </lightning-datatable>  
        </div>
    </lightning-card>
</template>

JavaScript:

import { 
    LightningElement, 
    api, 
    wire 
} from 'lwc';
import { getRelatedListRecords } from 'lightning/uiRelatedListApi';
import { NavigationMixin } from 'lightning/navigation';
import { EnclosingTabId, openSubtab, openTab } from 'lightning/platformWorkspaceApi';
const actions = [
    { label: 'New Tab', name: 'Newtab' },
    { label: 'Subtab', name: 'Subtab' }
];
const columns =  [   
    { label: 'Case Number', fieldName: 'CaseNumber' },
    { label: 'Subject', fieldName: 'Subject' },
    { label: 'Status', fieldName: 'Status' },
    {
        type: 'action',
        typeAttributes: { rowActions: actions }
    }
];

export default class RelatedCases extends NavigationMixin( LightningElement ) {
    
    columns = columns;

    @api recordId;
    records;    

    @wire(EnclosingTabId) 
    enclosingTabId;

    @wire( getRelatedListRecords, {

        parentRecordId: '$recordId',
        relatedListId: 'Cases',
        fields: [ 
            'Case.Id', 
            'Case.CaseNumber', 
            'Case.Subject' , 
            'Case.Status' 
        ]

    } )listInfo( { error, data } ) {

        if ( data ) {

            console.log( 
                'Data is', 
                JSON.stringify( data ) 
            );
            let tempRecords = [];

            data.records.forEach( obj => {

                let tempRecord = {};
                tempRecord.Id = obj.fields.Id.value;
                tempRecord.CaseNumber = obj.fields.CaseNumber.value;
                tempRecord.Subject = obj.fields.Subject.value;
                tempRecord.Status = obj.fields.Status.value;
                tempRecords.push( tempRecord );

            } );

            this.records = tempRecords;
            
        } else if (error) {
            
            this.records = undefined;

        }
    }
    
    handleRowAction( event ) {

        const row = event.detail.row;
        const actionName = event.detail.action.name;             
        
        switch ( actionName ) {

            case 'Newtab': 
                openTab( {
                    pageReference: {
                        type: "standard__recordPage",
                        attributes: {
                            recordId: row.Id,
                            actionName: 'view'
                        }
                    }
                } );
                break;
            case 'Subtab':                
                if ( !this.enclosingTabId ) {
                    return;
                } 
                openSubtab( this.enclosingTabId, {
                    pageReference: {
                        type: "standard__recordPage",
                        attributes: {
                            recordId: row.Id,
                            actionName: 'view'
                        }
                    }
                } );
                break;
            default:

        }        

    }

}

js-meta.xml:

<?xml version="1.0" encoding="UTF-8"?>
<LightningComponentBundle xmlns="http://soap.sforce.com/2006/04/metadata">
    <apiVersion>59.0</apiVersion>
    <isExposed>true</isExposed>
    <targets>
        <target>lightning__RecordPage</target>
    </targets>
</LightningComponentBundle>